datly 0.0.1 → 0.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datly",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "A JavaScript toolkit for data science, statistics, and machine learning in the browser or Node.js.",
   "keywords": [
     "data-science",
@@ -29,23 +29,36 @@
     "url": "https://github.com/vbfs/datly/issues"
   },
   "type": "module",
-  "main": "dist/datly.umd.js",
+  "main": "dist/datly.cjs",
+  "module": "dist/datly.mjs",
   "unpkg": "dist/datly.umd.js",
+  "exports": {
+    ".": {
+      "import": "./dist/datly.mjs",
+      "require": "./dist/datly.cjs",
+      "default": "./dist/datly.umd.js"
+    }
+  },
   "scripts": {
-    "build": "microbundle build --entry src/index.js --output dist/datly.umd.js --name Datly --format umd --compress --no-sourcemap",
-    "dev": "microbundle watch --entry src/index.js --output dist/datly.umd.js --name Datly --format umd",
+    "build": "microbundle --entry src/index.js --format modern,esm,cjs,umd --name Datly --compress --no-sourcemap --external none",
+    "dev": "microbundle watch --entry src/index.js --format modern,esm,cjs,umd --name Datly --external none",
     "prepublishOnly": "npm run build"
   },
   "devDependencies": {
-    "d3": "^7.9.0",
-    "microbundle": "^0.15.1"
+    "microbundle": "^0.15.1",
+    "d3-array": "^3.2.4",
+    "d3-axis": "^3.0.0",
+    "d3-scale": "^4.0.2",
+    "d3-scale-chromatic": "^3.1.0",
+    "d3-selection": "^3.0.0",
+    "d3-shape": "^3.2.0",
+    "d3-interpolate": "^3.0.1"
   },
+  "dependencies": {},
   "files": [
     "dist",
+    "src",
     "README.md",
     "LICENSE"
-  ],
-  "engines": {
-    "node": ">=14"
-  }
-}
+  ]
+}

package/src/core/dataLoader.js

@@ -0,0 +1,407 @@
+// Detecção de ambiente
+const isNode = typeof process !== 'undefined' &&
+               process.versions != null &&
+               process.versions.node != null;
+
+// Lazy load do fs
+let fs = null;
+let fsPromise = null;
+
+async function getFS() {
+  if (!isNode) return null;
+  if (fs) return fs;
+  if (fsPromise) return fsPromise;
+
+  fsPromise = import('fs').then(m => {
+    fs = m.default || m;
+    return fs;
+  }).catch(() => null);
+
+  return fsPromise;
+}
+
+class DataLoader {
+  async loadCSV(filePath, options = {}) {
+    const defaultOptions = {
+      delimiter: ',',
+      header: true,
+      skipEmptyLines: true,
+      encoding: 'utf8'
+    };
+
+    const config = { ...defaultOptions, ...options };
+
+    try {
+      let content;
+
+      // Node.js
+      if (isNode) {
+        const fsModule = await getFS();
+        if (fsModule) {
+          content = fsModule.readFileSync(filePath, { encoding: config.encoding });
+        }
+      }
+      // Browser com File System Access API
+      if (!content && typeof window !== 'undefined' && window.fs) {
+        content = window.fs.readFileSync
+          ? window.fs.readFileSync(filePath, { encoding: config.encoding })
+          : await window.fs.readFile(filePath, { encoding: config.encoding });
+      }
+      // Fetch API (browser)
+      if (!content && typeof fetch !== 'undefined') {
+        const response = await fetch(filePath);
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        content = await response.text();
+      }
+
+      if (!content) {
+        throw new Error('No file system available. Use fetch, fs module, or pass CSV text directly to parseCSV()');
+      }
+
+      return this.parseCSV(content, config);
+    } catch (error) {
+      throw new Error(`Failed to load CSV: ${error.message}`);
+    }
+  }
+
+  async loadJSON(jsonInput, options = {}) {
+    const defaultOptions = {
+      validateTypes: true,
+      autoInferHeaders: true
+    };
+
+    const config = { ...defaultOptions, ...options };
+
+    try {
+      let jsonData;
+
+      // Se é uma string que parece ser um caminho de arquivo
+      if (typeof jsonInput === 'string' && !jsonInput.trim().startsWith('{') && !jsonInput.trim().startsWith('[')) {
+        let content;
+
+        // Node.js
+        if (isNode) {
+          const fsModule = await getFS();
+          if (fsModule) {
+            content = fsModule.readFileSync(jsonInput, { encoding: 'utf8' });
+          }
+        }
+        // Browser com File System Access API
+        if (!content && typeof window !== 'undefined' && window.fs) {
+          content = window.fs.readFileSync
+            ? window.fs.readFileSync(jsonInput, { encoding: 'utf8' })
+            : await window.fs.readFile(jsonInput, { encoding: 'utf8' });
+        }
+        // Fetch API (browser)
+        if (!content && typeof fetch !== 'undefined') {
+          const response = await fetch(jsonInput);
+          if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+          }
+          content = await response.text();
+        }
+
+        if (!content) {
+          throw new Error('No file system available');
+        }
+
+        jsonData = JSON.parse(content);
+      }
+      // String JSON
+      else if (typeof jsonInput === 'string') {
+        jsonData = JSON.parse(jsonInput);
+      }
+      // Objeto JS
+      else if (typeof jsonInput === 'object') {
+        jsonData = jsonInput;
+      }
+      else {
+        throw new Error('Invalid JSON input: must be string, file path, or object');
+      }
+
+      return this.parseJSON(jsonData, config);
+    } catch (error) {
+      throw new Error(`Failed to load JSON: ${error.message}`);
+    }
+  }
+
+  parseJSON(jsonData, config) {
+    if (!jsonData) {
+      throw new Error('JSON data is empty or null');
+    }
+
+    if (Array.isArray(jsonData)) {
+      return this.parseJSONArray(jsonData, config);
+    } else if (jsonData.headers && jsonData.data) {
+      return this.parseStructuredJSON(jsonData, config);
+    } else if (typeof jsonData === 'object') {
+      return this.parseJSONObject(jsonData, config);
+    } else {
+      throw new Error('Unsupported JSON format');
+    }
+  }
+
+  parseJSONArray(jsonArray, config) {
+    if (jsonArray.length === 0) {
+      throw new Error('JSON array is empty');
+    }
+
+    const firstRow = jsonArray[0];
+    if (typeof firstRow !== 'object' || firstRow === null) {
+      throw new Error('JSON array must contain objects');
+    }
+
+    let headers;
+    if (config.autoInferHeaders) {
+      const allKeys = new Set();
+      jsonArray.forEach(row => {
+        if (typeof row === 'object' && row !== null) {
+          Object.keys(row).forEach(key => allKeys.add(key));
+        }
+      });
+      headers = Array.from(allKeys);
+    } else {
+      headers = Object.keys(firstRow);
+    }
+
+    const data = jsonArray.map((row, index) => {
+      if (typeof row !== 'object' || row === null) {
+        console.warn(`Row ${index} is not an object, skipping`);
+        return null;
+      }
+
+      const processedRow = {};
+      headers.forEach(header => {
+        let value = row[header];
+        if (config.validateTypes) {
+          value = this.inferType(value);
+        }
+        processedRow[header] = value;
+      });
+      return processedRow;
+    }).filter(row => row !== null);
+
+    return {
+      headers,
+      data,
+      length: data.length,
+      columns: headers.length,
+      source: 'json_array'
+    };
+  }
+
+  parseStructuredJSON(jsonData, config) {
+    const { headers, data } = jsonData;
+
+    if (!Array.isArray(headers)) {
+      throw new Error('Headers must be an array');
+    }
+
+    if (!Array.isArray(data)) {
+      throw new Error('Data must be an array');
+    }
+
+    if (headers.length === 0) {
+      throw new Error('Headers array is empty');
+    }
+
+    const processedData = data.map((row, index) => {
+      if (Array.isArray(row)) {
+        const processedRow = {};
+        headers.forEach((header, i) => {
+          let value = i < row.length ? row[i] : null;
+          if (config.validateTypes) {
+            value = this.inferType(value);
+          }
+          processedRow[header] = value;
+        });
+        return processedRow;
+      } else if (typeof row === 'object' && row !== null) {
+        const processedRow = {};
+        headers.forEach(header => {
+          let value = row[header];
+          if (config.validateTypes) {
+            value = this.inferType(value);
+          }
+          processedRow[header] = value;
+        });
+        return processedRow;
+      } else {
+        console.warn(`Row ${index} has invalid format, skipping`);
+        return null;
+      }
+    }).filter(row => row !== null);
+
+    return {
+      headers,
+      data: processedData,
+      length: processedData.length,
+      columns: headers.length,
+      source: 'structured_json'
+    };
+  }
+
+  parseJSONObject(jsonObject, config) {
+    const entries = Object.entries(jsonObject);
+    if (entries.length === 0) {
+      throw new Error('JSON object is empty');
+    }
+
+    const headers = ['key', 'value'];
+    const data = entries.map(([key, value]) => ({
+      key: key,
+      value: config.validateTypes ? this.inferType(value) : value
+    }));
+
+    return {
+      headers,
+      data,
+      length: data.length,
+      columns: 2,
+      source: 'json_object'
+    };
+  }
+
+  parseCSV(content, options) {
+    const lines = content.split('\n').filter(line =>
+      options.skipEmptyLines ? line.trim() !== '' : true
+    );
+
+    if (lines.length === 0) {
+      throw new Error('CSV file is empty');
+    }
+
+    const headers = options.header
+      ? lines[0].split(options.delimiter).map(h => h.trim().replace(/['"]/g, ''))
+      : Array.from({ length: lines[0].split(options.delimiter).length }, (_, i) => `col_${i}`);
+
+    const startIndex = options.header ? 1 : 0;
+    const data = [];
+
+    for (let i = startIndex; i < lines.length; i++) {
+      const values = lines[i].split(options.delimiter);
+      if (values.length === headers.length) {
+        const row = {};
+        headers.forEach((header, index) => {
+          let value = values[index].trim().replace(/['"]/g, '');
+          row[header] = this.inferType(value);
+        });
+        data.push(row);
+      }
+    }
+
+    return {
+      headers,
+      data,
+      length: data.length,
+      columns: headers.length
+    };
+  }
+
+  inferType(value) {
+    if (value === '' || value === 'null' || value === 'NULL' || value === 'NaN') {
+      return null;
+    }
+
+    if (value === 'true' || value === 'TRUE') return true;
+    if (value === 'false' || value === 'FALSE') return false;
+
+    if (/^-?\d+$/.test(value)) {
+      return parseInt(value, 10);
+    }
+
+    if (/^-?\d*\.\d+$/.test(value)) {
+      return parseFloat(value);
+    }
+
+    return value;
+  }
+
+  cleanData(dataset) {
+    const cleaned = {
+      ...dataset,
+      data: dataset.data.filter(row => {
+        return Object.values(row).some(value => value !== null && value !== undefined);
+      })
+    };
+
+    cleaned.length = cleaned.data.length;
+    return cleaned;
+  }
+
+  getDataInfo(dataset) {
+    const info = {
+      rows: dataset.length,
+      columns: dataset.columns,
+      headers: dataset.headers,
+      types: {},
+      nullCounts: {},
+      uniqueCounts: {}
+    };
+
+    dataset.headers.forEach(header => {
+      const column = dataset.data.map(row => row[header]);
+      const nonNullValues = column.filter(val => val !== null && val !== undefined);
+      const types = [...new Set(nonNullValues.map(val => typeof val))];
+
+      info.types[header] = types.length === 1 ? types[0] : 'mixed';
+      info.nullCounts[header] = column.length - nonNullValues.length;
+      info.uniqueCounts[header] = new Set(nonNullValues).size;
+    });
+
+    return info;
+  }
+
+  getColumn(dataset, columnName) {
+    if (!dataset.headers.includes(columnName)) {
+      throw new Error(`Column '${columnName}' not found`);
+    }
+
+    return dataset.data
+      .map(row => row[columnName])
+      .filter(val => val !== null && val !== undefined && !isNaN(val));
+  }
+
+  getColumns(dataset, columnNames) {
+    const result = {};
+    columnNames.forEach(name => {
+      result[name] = this.getColumn(dataset, name);
+    });
+    return result;
+  }
+
+  filterRows(dataset, condition) {
+    return {
+      ...dataset,
+      data: dataset.data.filter(condition),
+      length: dataset.data.filter(condition).length
+    };
+  }
+
+  sortBy(dataset, columnName, order = 'asc') {
+    const sortedData = [...dataset.data].sort((a, b) => {
+      const aVal = a[columnName];
+      const bVal = b[columnName];
+
+      if (aVal === null || aVal === undefined) return 1;
+      if (bVal === null || bVal === undefined) return -1;
+
+      if (typeof aVal === 'string' && typeof bVal === 'string') {
+        return order === 'asc'
+          ? aVal.localeCompare(bVal)
+          : bVal.localeCompare(aVal);
+      }
+
+      return order === 'asc' ? aVal - bVal : bVal - aVal;
+    });
+
+    return {
+      ...dataset,
+      data: sortedData
+    };
+  }
+}
+
+export default DataLoader;