meadow-integration 1.0.4 → 1.0.6

package/docs/rest/csv.md

@@ -0,0 +1,255 @@
+# CSV Endpoints
+
+## POST /1.0/CSV/Check
+
+Analyze a CSV file and return statistics about its structure and content.
+
+### Request Body
+
+| Property | Type | Required | Default | Description |
+|----------|------|----------|---------|-------------|
+| `File` | string | Yes | - | Absolute path to the CSV file to analyze |
+| `Records` | boolean | No | `false` | When `true`, include all parsed records in the response |
+| `QuoteDelimiter` | string | No | `"` | Character used for quoting fields |
+
+### Response
+
+A statistics object containing row count, column count, column names, and optionally the full parsed records.
+
+```json
+{
+    "File": "/data/books.csv",
+    "RowCount": 42,
+    "ColumnCount": 5,
+    "Columns": ["id", "title", "author", "year", "genre"],
+    "Records": []
+}
+```
+
+### Error Responses
+
+| Status | Condition |
+|--------|-----------|
+| 400 | No valid `File` path provided |
+| 404 | File does not exist |
+| 500 | Error reading the CSV file |
+
+### curl Example
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/CSV/Check \
+  -H "Content-Type: application/json" \
+  -d '{
+    "File": "/data/books.csv"
+  }'
+```
+
+With records included:
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/CSV/Check \
+  -H "Content-Type: application/json" \
+  -d '{
+    "File": "/data/books.csv",
+    "Records": true
+  }'
+```
+
+### JavaScript Example (dependency-free)
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    File: '/data/books.csv',
+    Records: false
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/CSV/Check',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const stats = JSON.parse(data);
+        console.log(`Rows: ${stats.RowCount}, Columns: ${stats.ColumnCount}`);
+        console.log('Column names:', stats.Columns);
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```
+
+---
+
+## POST /1.0/CSV/Transform
+
+Transform a CSV file into a comprehension object. A comprehension is a GUID-keyed hash of entity records used throughout the Meadow ecosystem for data integration.
+
+### Request Body
+
+| Property | Type | Required | Default | Description |
+|----------|------|----------|---------|-------------|
+| `File` | string | Yes | - | Absolute path to the CSV file to transform |
+| `Entity` | string | No | Derived from filename | Entity name for the comprehension |
+| `GUIDName` | string | No | `GUID{Entity}` | Name of the GUID column in the output |
+| `GUIDTemplate` | string | No | Auto-generated | Pict template expression for generating GUIDs (e.g. `{~D:Record.id~}`) |
+| `Mappings` | object | No | - | Column-level mapping overrides (`{ "OutputCol": "{~D:Record.input_col~}" }`) |
+| `MappingConfiguration` | object | No | - | Full explicit mapping configuration object |
+| `IncomingComprehension` | object | No | `{}` | Existing comprehension to merge new records into |
+| `Extended` | boolean | No | `false` | When `true`, return the full mapping outcome state instead of just the comprehension |
+| `QuoteDelimiter` | string | No | `"` | Character used for quoting fields |
+
+### Response
+
+By default, returns the comprehension object:
+
+```json
+{
+    "Book": {
+        "0x42": {
+            "GUIDBook": "0x42",
+            "id": "1",
+            "title": "The Great Gatsby",
+            "author": "F. Scott Fitzgerald"
+        }
+    }
+}
+```
+
+When `Extended` is `true`, the response includes the full mapping outcome with `Comprehension`, `Configuration`, `ImplicitConfiguration`, `UserConfiguration`, `ParsedRowCount`, and more.
+
+### Error Responses
+
+| Status | Condition |
+|--------|-----------|
+| 400 | No valid `File` path provided |
+| 404 | File does not exist |
+| 500 | Error reading the CSV file |
+
+### curl Example
+
+Basic transform:
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/CSV/Transform \
+  -H "Content-Type: application/json" \
+  -d '{
+    "File": "/data/books.csv"
+  }'
+```
+
+With custom entity name and GUID template:
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/CSV/Transform \
+  -H "Content-Type: application/json" \
+  -d '{
+    "File": "/data/books.csv",
+    "Entity": "Book",
+    "GUIDTemplate": "{~D:Record.id~}",
+    "Extended": true
+  }'
+```
+
+With column mappings:
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/CSV/Transform \
+  -H "Content-Type: application/json" \
+  -d '{
+    "File": "/data/books.csv",
+    "Entity": "Book",
+    "Mappings": {
+      "BookTitle": "{~D:Record.title~}",
+      "AuthorName": "{~D:Record.author~}"
+    }
+  }'
+```
+
+### JavaScript Example (dependency-free)
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    File: '/data/books.csv',
+    Entity: 'Book',
+    GUIDTemplate: '{~D:Record.id~}'
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/CSV/Transform',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const comprehension = JSON.parse(data);
+        const entityKeys = Object.keys(comprehension);
+        console.log('Entities:', entityKeys);
+        entityKeys.forEach((entity) => {
+            const recordCount = Object.keys(comprehension[entity]).length;
+            console.log(`  ${entity}: ${recordCount} records`);
+        });
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```
+
+### Extended Response Example
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    File: '/data/books.csv',
+    Entity: 'Book',
+    Extended: true
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/CSV/Transform',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const outcome = JSON.parse(data);
+        console.log('Parsed rows:', outcome.ParsedRowCount);
+        console.log('Configuration:', JSON.stringify(outcome.Configuration, null, 2));
+        console.log('Comprehension keys:', Object.keys(outcome.Comprehension));
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```

package/docs/rest/entity-generation.md

@@ -0,0 +1,158 @@
+# Entity Generation Endpoint
+
+## POST /1.0/Entity/FromTabularFolder
+
+Generate a combined comprehension from all tabular data files (CSV, TSV, and JSON) found in a folder. Each file becomes an entity in the output comprehension, with the entity name derived from the filename (without extension) unless overridden.
+
+### Request Body
+
+| Property | Type | Required | Default | Description |
+|----------|------|----------|---------|-------------|
+| `Folder` | string | Yes | - | Absolute path to a folder containing tabular files |
+| `Entity` | string | No | Derived from each filename | Force all files to use the same entity name |
+| `MappingConfiguration` | object | No | - | Mapping hints configuration applied to all files |
+
+### Supported File Types
+
+| Extension | Format |
+|-----------|--------|
+| `.csv` | Comma-separated values |
+| `.tsv` | Tab-separated values |
+| `.json` | JSON array of objects |
+
+Files with other extensions are silently ignored.
+
+### Response
+
+Returns a comprehension object containing all entities generated from files in the folder:
+
+```json
+{
+    "housing_costs_Neighborhoods": {
+        "0x1": { "GUIDhousing_costs_Neighborhoods": "0x1", "Neighborhood": "Ballard", "MedianRent": "1450" }
+    },
+    "race_ethnicity_Neighborhoods": {
+        "0x1": { "GUIDrace_ethnicity_Neighborhoods": "0x1", "Neighborhood": "Ballard", "White": "78.2" }
+    }
+}
+```
+
+### Error Responses
+
+| Status | Condition |
+|--------|-----------|
+| 400 | No valid `Folder` path provided, path is not a directory, or no tabular files found |
+| 404 | Folder does not exist |
+| 500 | Error processing files |
+
+### curl Example
+
+Basic folder processing:
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/Entity/FromTabularFolder \
+  -H "Content-Type: application/json" \
+  -d '{
+    "Folder": "/data/seattle_neighborhoods/"
+  }'
+```
+
+With a forced entity name:
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/Entity/FromTabularFolder \
+  -H "Content-Type: application/json" \
+  -d '{
+    "Folder": "/data/imports/",
+    "Entity": "ImportedRecord"
+  }'
+```
+
+### JavaScript Example (dependency-free)
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    Folder: '/data/seattle_neighborhoods/'
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/Entity/FromTabularFolder',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const comprehension = JSON.parse(data);
+        const entities = Object.keys(comprehension);
+        console.log(`Generated ${entities.length} entity(ies):`);
+        entities.forEach((entity) => {
+            const recordCount = Object.keys(comprehension[entity]).length;
+            console.log(`  ${entity}: ${recordCount} records`);
+        });
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```
+
+### With Mapping Configuration
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    Folder: '/data/imports/',
+    Entity: 'Product',
+    MappingConfiguration: {
+        GUIDTemplate: '{~D:Record.sku~}',
+        Mappings: {
+            ProductName: '{~D:Record.name~}',
+            UnitPrice: '{~D:Record.price~}'
+        }
+    }
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/Entity/FromTabularFolder',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const comprehension = JSON.parse(data);
+        console.log('Products generated:', JSON.stringify(comprehension, null, 2));
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```
+
+### How It Works
+
+1. The endpoint reads all files in the specified folder.
+2. Files are filtered to supported extensions (`.csv`, `.tsv`, `.json`).
+3. Each file is processed asynchronously using the `anticipate` pattern.
+4. For CSV and TSV files, a fresh parser is created per file with the appropriate delimiter.
+5. For JSON files, the contents are parsed and validated as an array.
+6. Entity names are derived from the filename (without extension) unless `Entity` is provided.
+7. Results from all files are merged into a single comprehension object.

package/docs/rest/json-array.md

@@ -0,0 +1,243 @@
+# JSON Array Endpoints
+
+## POST /1.0/JSONArray/Transform
+
+Transform a JSON array file into a comprehension object. The file must contain a valid JSON array of objects.
+
+### Request Body
+
+| Property | Type | Required | Default | Description |
+|----------|------|----------|---------|-------------|
+| `File` | string | Yes | - | Absolute path to a JSON file containing an array of objects |
+| `Entity` | string | No | Derived from filename | Entity name for the comprehension |
+| `GUIDName` | string | No | `GUID{Entity}` | Name of the GUID column in the output |
+| `GUIDTemplate` | string | No | Auto-generated | Pict template expression for generating GUIDs |
+| `Mappings` | object | No | - | Column-level mapping overrides |
+| `MappingConfiguration` | object | No | - | Full explicit mapping configuration object |
+| `IncomingComprehension` | object | No | `{}` | Existing comprehension to merge new records into |
+| `Extended` | boolean | No | `false` | Return full mapping outcome state |
+
+### Response
+
+Returns a comprehension object:
+
+```json
+{
+    "Customer": {
+        "CUST-100": {
+            "GUIDCustomer": "CUST-100",
+            "id": 100,
+            "name": "Acme Corp",
+            "email": "contact@acme.com"
+        }
+    }
+}
+```
+
+### Error Responses
+
+| Status | Condition |
+|--------|-----------|
+| 400 | No valid `File` path provided, file is not valid JSON, or file does not contain an array |
+| 404 | File does not exist |
+
+### curl Example
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/JSONArray/Transform \
+  -H "Content-Type: application/json" \
+  -d '{
+    "File": "/data/customers.json",
+    "Entity": "Customer",
+    "GUIDTemplate": "{~D:Record.id~}"
+  }'
+```
+
+### JavaScript Example (dependency-free)
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    File: '/data/customers.json',
+    Entity: 'Customer',
+    GUIDTemplate: '{~D:Record.id~}'
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/JSONArray/Transform',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const comprehension = JSON.parse(data);
+        const entityKeys = Object.keys(comprehension);
+        console.log('Entities:', entityKeys);
+        entityKeys.forEach((entity) => {
+            const recordCount = Object.keys(comprehension[entity]).length;
+            console.log(`  ${entity}: ${recordCount} records`);
+        });
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```
+
+---
+
+## POST /1.0/JSONArray/TransformRecords
+
+Transform an in-memory JSON array of records into a comprehension object. Unlike `/JSONArray/Transform`, this endpoint does not require a file on disk -- the records are passed directly in the request body.
+
+### Request Body
+
+| Property | Type | Required | Default | Description |
+|----------|------|----------|---------|-------------|
+| `Records` | array | Yes | - | Array of record objects to transform |
+| `Entity` | string | No | `"Records"` | Entity name for the comprehension |
+| `GUIDName` | string | No | `GUID{Entity}` | Name of the GUID column in the output |
+| `GUIDTemplate` | string | No | Auto-generated | Pict template expression for generating GUIDs |
+| `Mappings` | object | No | - | Column-level mapping overrides |
+| `MappingConfiguration` | object | No | - | Full explicit mapping configuration object |
+| `IncomingComprehension` | object | No | `{}` | Existing comprehension to merge new records into |
+| `Extended` | boolean | No | `false` | Return full mapping outcome state |
+
+### Response
+
+Returns a comprehension object:
+
+```json
+{
+    "Order": {
+        "ORD-001": {
+            "GUIDOrder": "ORD-001",
+            "order_id": "ORD-001",
+            "customer": "Acme Corp",
+            "total": 250.00
+        }
+    }
+}
+```
+
+### Error Responses
+
+| Status | Condition |
+|--------|-----------|
+| 400 | No valid `Records` array provided or array is empty |
+
+### curl Example
+
+```bash
+curl -s -X POST http://localhost:8086/1.0/JSONArray/TransformRecords \
+  -H "Content-Type: application/json" \
+  -d '{
+    "Records": [
+      { "order_id": "ORD-001", "customer": "Acme Corp", "total": 250.00 },
+      { "order_id": "ORD-002", "customer": "Globex Inc", "total": 175.50 }
+    ],
+    "Entity": "Order",
+    "GUIDTemplate": "{~D:Record.order_id~}"
+  }'
+```
+
+### JavaScript Example (dependency-free)
+
+```javascript
+const http = require('http');
+
+const requestBody = JSON.stringify({
+    Records: [
+        { order_id: 'ORD-001', customer: 'Acme Corp', total: 250.00 },
+        { order_id: 'ORD-002', customer: 'Globex Inc', total: 175.50 },
+        { order_id: 'ORD-003', customer: 'Initech', total: 320.00 }
+    ],
+    Entity: 'Order',
+    GUIDTemplate: '{~D:Record.order_id~}'
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/JSONArray/TransformRecords',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const comprehension = JSON.parse(data);
+        console.log('Order comprehension:');
+        const orders = comprehension.Order;
+        Object.keys(orders).forEach((guid) => {
+            console.log(`  ${guid}: ${orders[guid].customer} - $${orders[guid].total}`);
+        });
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```
+
+### In-Memory Pipeline Example
+
+This example shows how to use `TransformRecords` to convert data already in memory (e.g. from an API response) into a comprehension without writing to disk:
+
+```javascript
+const http = require('http');
+
+// Simulate records from an external API
+const externalData = [
+    { id: 1, name: 'Alice', department: 'Engineering' },
+    { id: 2, name: 'Bob', department: 'Marketing' },
+    { id: 3, name: 'Carol', department: 'Engineering' }
+];
+
+const requestBody = JSON.stringify({
+    Records: externalData,
+    Entity: 'Employee',
+    GUIDTemplate: '{~D:Record.id~}',
+    Mappings: {
+        FullName: '{~D:Record.name~}',
+        Dept: '{~D:Record.department~}'
+    }
+});
+
+const options = {
+    hostname: 'localhost',
+    port: 8086,
+    path: '/1.0/JSONArray/TransformRecords',
+    method: 'POST',
+    headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(requestBody)
+    }
+};
+
+const req = http.request(options, (res) => {
+    let data = '';
+    res.on('data', (chunk) => { data += chunk; });
+    res.on('end', () => {
+        const comprehension = JSON.parse(data);
+        console.log('Employee comprehension created with',
+            Object.keys(comprehension.Employee).length, 'records');
+    });
+});
+req.on('error', (err) => { console.error('Request error:', err.message); });
+req.write(requestBody);
+req.end();
+```