meadow-integration 1.0.4 → 1.0.6

package/.dockerignore

@@ -0,0 +1,11 @@
+.*
+coverage
+test
+node_modules
+package-lock.json
+Docker-*.sh
+*.yml
+.vscode
+logs
+*.patch
+debug

package/Docker-Build.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker build -t retold/meadow-integration -f Dockerfile .

package/Docker-Compose.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker-compose -f docker-compose.yml up

package/Docker-Push.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker push retold/meadow-integration

package/Docker-Tag.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker tag retold/meadow-integration retold/meadow-integration:$1

package/Dockerfile

@@ -0,0 +1,28 @@
+FROM node:20-bookworm AS base
+MAINTAINER steven velozo <steven@velozo.com>
+
+RUN apt-get update && apt-get -y --force-yes install curl vim nano less \
+	tmux uuid-runtime
+
+# Setup node.js global tools
+RUN npm install -g nodemon --unsafe-perm
+
+ADD package.json /service_root/package.json
+RUN cd /service_root && npm install --omit=dev
+
+ADD source /service_root/source
+ADD scripts /service_root/scripts
+
+WORKDIR /service_root
+
+# Don't leave local artifacts around
+RUN rm -rf package-lock.json .git test
+
+# Default config file for Docker deployments
+RUN if [ -f ./Meadow-Config-Docker.json ]; then mv ./Meadow-Config-Docker.json ./source/cli/Default-Meadow-Integration-Configuration.json; fi
+
+FROM base AS production
+
+RUN date -u +"%Y-%m-%dT%H:%M:%SZ" > ./build.date
+
+CMD ["scripts/run.sh"]

package/Dockerfile_LUXURYCode

@@ -0,0 +1,23 @@
+FROM codercom/code-server:latest
+
+USER root
+
+RUN apt-get update && apt-get install -y vim curl tmux
+
+# Install NVM / Node
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
+RUN bash -c "source /root/.nvm/nvm.sh && nvm install 20 && nvm alias default 20"
+
+# Install VSCode extensions
+RUN code-server --install-extension hbenl.vscode-mocha-test-adapter
+RUN code-server --install-extension hbenl.vscode-test-adapter-converter
+RUN code-server --install-extension hbenl.vscode-test-explorer
+RUN code-server --install-extension oderwat.indent-rainbow
+RUN code-server --install-extension dbaeumer.vscode-eslint
+RUN code-server --install-extension eamodio.gitlens
+
+VOLUME ["/home/coder/.config"]
+VOLUME ["/home/coder/.vscode"]
+VOLUME ["/home/coder/meadow-integration"]
+
+WORKDIR /home/coder/meadow-integration

package/README.md

@@ -51,19 +51,30 @@ npm start -- csvtransform ./docs/examples/data/books.csv \
 npm start -- comprehensionintersect Set1.json -i Set2.json -e "MyEntity" -o merged.json
 ```
 
+### Clone Data from a Remote API
+
+```shell
+npm start -- data-clone \
+  --api_server "https://my-meadow-api.example.com/1.0/" \
+  --api_username admin --api_password secret \
+  --db_host 127.0.0.1 --db_name my_local_db \
+  --schema_path ./schema/Model-Extended.json
+```
+
 ## CLI Commands
 
-| Command | Description |
-|---------|-------------|
-| `csvcheck` | Analyze a CSV file for statistics |
-| `csvtransform` | Transform a CSV into a comprehension |
-| `tsvtransform` | Transform a TSV into a comprehension |
-| `jsonarraytransform` | Transform a JSON array into a comprehension |
-| `comprehensionintersect` | Merge two comprehension files |
-| `comprehensionarray` | Convert object comprehension to array |
-| `objectarraytocsv` | Export a JSON array to CSV |
-| `load_comprehension` | Push a comprehension to Meadow REST APIs |
-| `serve` | Start the REST API server |
+| Command | Aliases | Description |
+|---------|---------|-------------|
+| `csvcheck` | | Analyze a CSV file for statistics |
+| `csvtransform` | | Transform a CSV into a comprehension |
+| `tsvtransform` | | Transform a TSV into a comprehension |
+| `jsonarraytransform` | | Transform a JSON array into a comprehension |
+| `comprehensionintersect` | | Merge two comprehension files |
+| `comprehensionarray` | | Convert object comprehension to array |
+| `objectarraytocsv` | | Export a JSON array to CSV |
+| `load_comprehension` | | Push a comprehension to Meadow REST APIs |
+| `data-clone` | `clone`, `sync` | Clone data from a Meadow API to a local database |
+| `serve` | | Start the REST API server |
 
 ## REST API Server
 
@@ -178,24 +189,124 @@ For multi-record generation (e.g. splitting comma-separated authors), use Solver
 ## Architecture
 
 ```
-External Data (CSV / TSV / JSON)
-        |
-        v
-   TabularTransform Service  -- column mapping via Pict templates
-        |
-        v
-   Comprehension Object      -- entity records keyed by GUID
-        |
-        v
-   Integration Adapter       -- marshal to Meadow schema
-        |
-        v
-   GUID Map                  -- track external <-> Meadow IDs
+External Data (CSV / TSV / JSON)          Meadow REST API (Source)
+        |                                         |
+        v                                         v
+   TabularTransform Service               CloneRestClient Service
+   -- column mapping via Pict templates    -- authenticate & read entities
+        |                                         |
+        v                                         v
+   Comprehension Object                    Sync Service (Initial / Ongoing)
+   -- entity records keyed by GUID         -- compare server vs local state
+        |                                         |
+        v                                         v
+   Integration Adapter                     ConnectionManager Service
+   -- marshal to Meadow schema             -- MySQL or MSSQL connection pool
+        |                                         |
+        v                                         v
+   GUID Map                                Local Database
+   -- track external <-> Meadow IDs        -- tables created from schema
         |
         v
-   Meadow REST API           -- batch upsert / single upsert
+   Meadow REST API (Destination)
+   -- batch upsert / single upsert
+```
+
+## Data Synchronization
+
+The `data-clone` command synchronizes entity data from a remote Meadow REST API into a local MySQL or MSSQL database. It authenticates with the source API, connects to a local database, loads a Meadow schema, and syncs each entity defined in the schema.
+
+### Services
+
+| Service | Description |
+|---------|-------------|
+| **ConnectionManager** | Manages database connection pools for MySQL and MSSQL. Handles table creation and index management on the local database. |
+| **CloneRestClient** | Authenticates with a remote Meadow API and provides methods for reading, creating, updating, upserting, and deleting entities. Includes built-in entity caching and paginated batch downloads. |
+| **Sync** | Orchestrates entity synchronization. Loads a Meadow schema, creates local tables if they do not exist, and iterates through each entity to sync records. |
+
+### Sync Modes
+
+| Mode | Description |
+|------|-------------|
+| **Initial** | Compares the max entity ID on the server against the local database and downloads all records with IDs greater than the local maximum. Designed for first-time bulk population. |
+| **Ongoing** | Walks through all server records page by page, comparing `UpdateDate` timestamps. Creates missing records locally and updates records whose server timestamp is newer than the local copy. |
+
+### CLI Options
+
+```
+data-clone [options]
+
+Options:
+  -a, --api_server <url>        Source Meadow API server URL
+  -u, --api_username <user>     API username for authentication
+  -p, --api_password <pass>     API password for authentication
+  -d, --db_provider <provider>  Database provider: MySQL or MSSQL (default: MySQL)
+  -dh, --db_host <host>         Database host address
+  -dp, --db_port <port>         Database port
+  -du, --db_username <user>     Database username
+  -dw, --db_password <pass>     Database password
+  -dn, --db_name <name>         Database name
+  -sp, --schema_path <path>     Path to Meadow extended schema JSON file
+  -s, --sync_mode <mode>        Sync mode: Initial or Ongoing (default: Initial)
+  -w, --post_run_delay <min>    Minutes to wait after sync before exiting (default: 0)
+```
+
+### Configuration
+
+The `data-clone` command reads configuration from a `.meadow.config.json` file. The CLI searches for this file starting from the current working directory and cascading up to the home directory (powered by `pict-service-commandlineutility` auto-gather). Command-line options override values from the config file.
+
+```json
+{
+  "Source": {
+    "ServerURL": "https://my-meadow-api.example.com/1.0/",
+    "UserID": "admin",
+    "Password": "secret"
+  },
+  "Destination": {
+    "Provider": "MySQL",
+    "MySQL": {
+      "server": "127.0.0.1",
+      "port": 3306,
+      "user": "root",
+      "password": "",
+      "database": "meadow",
+      "connectionLimit": 20
+    },
+    "MSSQL": {
+      "server": "127.0.0.1",
+      "port": 1433,
+      "user": "sa",
+      "password": "",
+      "database": "meadow",
+      "ConnectionPoolLimit": 20
+    }
+  },
+  "SchemaPath": "./schema/Model-Extended.json",
+  "Sync": {
+    "DefaultSyncMode": "Initial",
+    "PageSize": 100,
+    "SyncEntityList": [],
+    "SyncEntityOptions": {}
+  }
+}
+```
+
+Set `SyncEntityList` to an array of entity names to sync only a subset of the schema. When empty, all entities in the schema are synced.
+
+### Docker
+
+A `Dockerfile` and `docker-compose.yml` are provided for running `data-clone` in a container. The Docker image is based on `node:20-bookworm` and runs the sync via the `scripts/run.sh` entrypoint.
+
+```shell
+# Build the image
+docker build -t retold/meadow-integration .
+
+# Run with docker-compose (connects to a meadow_backend network)
+docker-compose up
 ```
 
+Place a `.meadow.config.json` in the mounted volume or use `Meadow-Config-Docker.json` at the project root (it is automatically copied into the container as the default configuration during the Docker build).
+
 ## Documentation
 
 Full documentation is available at `docs/index.html` (powered by pict-docuserve):
@@ -219,8 +330,11 @@ npm test
 
 - [meadow](https://github.com/stevenvelozo/meadow) - Data access and ORM
 - [meadow-endpoints](https://github.com/stevenvelozo/meadow-endpoints) - Auto-generated REST endpoints
+- [meadow-connection-mysql](https://github.com/stevenvelozo/meadow-connection-mysql) - MySQL database provider for Meadow
+- [meadow-connection-mssql](https://github.com/stevenvelozo/meadow-connection-mssql) - MSSQL database provider for Meadow
 - [orator](https://github.com/stevenvelozo/orator) - API server abstraction
 - [fable](https://github.com/stevenvelozo/fable) - Application services framework
+- [pict-service-commandlineutility](https://github.com/stevenvelozo/pict-service-commandlineutility) - CLI framework with cascading configuration
 
 ## License
 

package/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '2'
+
+services:
+  meadow-integration-clone:
+    image: retold/meadow-integration:latest
+    volumes:
+      - "${RETOLD_DIR:-./}:/service_root"
+    environment:
+      - RUN_LOCAL_DEV=true
+    networks:
+      - back
+
+networks:
+  back:
+    external:
+      name: meadow_backend

package/docs/README.md

@@ -4,6 +4,8 @@ A suite of tools for managing data into a centralized non-specific schema format
 
 These tools are built to be usable from the command-line, as a web service, or within your own codebase.  This module presents these behaviors both as a suite of externally usable fable services, a command-line utility to leverage them and a set of web service behaviors.
 
+In addition to data transformation, Meadow Integration includes a **Data Clone** pipeline for replicating entity data from a remote Meadow REST API into a local MySQL or MSSQL database.
+
 ## What is a Comprehension?
 
 A **Comprehension** is a JSON object that stores entity records keyed by their GUID.  It acts as an intermediate data format for integrating records from external systems (CSV, TSV, JSON) into Meadow entities.
@@ -78,38 +80,83 @@ npx meadow-integration csvtransform ./books.csv -m mapping.json -o books.json
 npx meadow-integration comprehensionintersect Set1.json -i Set2.json -e "MyEntity" -o merged.json
 ```
 
+### 5. Clone data from a remote API
+
+Create a `.meadow.config.json` in your working directory:
+
+```json
+{
+  "Source": {
+    "ServerURL": "https://api.example.com/1.0/",
+    "UserID": "sync-user",
+    "Password": "sync-password"
+  },
+  "Destination": {
+    "Provider": "MySQL",
+    "MySQL": {
+      "server": "127.0.0.1",
+      "port": 3306,
+      "user": "root",
+      "password": "",
+      "database": "meadow_clone"
+    }
+  },
+  "SchemaPath": "./schema/Model-Extended.json"
+}
+```
+
+Then run the clone:
+
+```shell
+npx meadow-integration data-clone
+```
+
+For incremental updates after the initial clone:
+
+```shell
+npx meadow-integration data-clone -s Ongoing
+```
+
 ## Architecture
 
 ```
-External Data (CSV / TSV / JSON)
-        |
-        v
-   TabularTransform Service
-   (column mapping via Pict templates)
-        |
-        v
-   Comprehension Object
-   (Entity records keyed by GUID)
-        |
-        v
-   Integration Adapter
-   (marshal to Meadow schema)
-        |
-        v
-   GUID Map
-   (track external <-> Meadow IDs)
-        |
+External Data (CSV / TSV / JSON)         Remote Meadow REST API
+        |                                         |
+        v                                         v
+   TabularTransform Service              CloneRestClient
+   (column mapping via Pict templates)   (authenticated HTTP)
+        |                                         |
+        v                                         v
+   Comprehension Object                  Sync Service
+   (Entity records keyed by GUID)        (Initial or Ongoing mode)
+        |                                         |
+        v                                         v
+   Integration Adapter                   ConnectionManager
+   (marshal to Meadow schema)            (MySQL or MSSQL pool)
+        |                                         |
+        v                                         v
+   GUID Map                              Local Database
+   (track external <-> Meadow IDs)       (tables auto-created
+        |                                 from Meadow schema)
         v
    Meadow REST API
    (batch upsert / single upsert)
 ```
 
+The left side is the **Data Transformation** pipeline: external files are parsed, transformed into comprehensions, and pushed to a Meadow API. The right side is the **Data Synchronization** pipeline: entity data is pulled from a remote Meadow API and written to a local database.
+
+Both pipelines share the Fable service provider pattern for dependency injection, logging, and configuration.
+
 ## Next Steps
 
+- [Overview](overview.md) -- Full feature overview and when to use each tool
+- [Quick Start Guide](quickstart.md) -- Step-by-step walkthrough of all workflows
+- [Architecture](architecture.md) -- System design with mermaid diagrams
 - [CLI Reference](cli-reference.md) -- All commands and their options
 - [REST API Reference](rest-api-reference.md) -- All REST endpoints with curl examples
 - [Mapping Files](mapping-files.md) -- How to write column mapping configurations
 - [Comprehensions](comprehensions.md) -- The comprehension data format in detail
 - [Programmatic API](programmatic-api.md) -- Using services directly in your code
 - [Integration Adapter](integration-adapter.md) -- Pushing data to Meadow REST APIs
+- [Data Clone Overview](data-clone/overview.md) -- Synchronizing remote APIs to local databases
 - [Examples](examples-walkthrough.md) -- Walkthrough of all runnable examples

package/docs/{cover.md → _cover.md}

@@ -1,10 +1,11 @@
-# Meadow Integration <small>1.0</small>
+# Meadow Integration <small>1.1</small>
 
-> Data integration toolkit for transforming CSV, TSV, and JSON into Meadow entity comprehensions
+> Data integration toolkit for transforming, synchronizing, and managing data with Meadow entity comprehensions
 
 - Transform tabular data into structured entity records
 - Merge multiple data sources by shared keys
 - Multi-entity extraction from single sources using Solvers
+- Synchronize remote Meadow APIs to local databases
 - CLI utility, programmatic API, and REST integration
 
 [GitHub](https://github.com/stevenvelozo/meadow-integration)

package/docs/_sidebar.md

@@ -1,19 +1,64 @@
 - Getting Started
 
   - [Introduction](/)
-  - [Quick Start](README.md)
+  - [Quick Start](quickstart.md)
+  - [Overview](overview.md)
 
-- Concepts
+- Architecture
+
+  - [Design](architecture.md)
+  - [Implementation Reference](implementation-reference.md)
+
+- Data Transformation
 
   - [Comprehensions](comprehensions.md)
   - [Mapping Files](mapping-files.md)
 
-- Reference
+- Data Synchronization
+
+  - [Data Clone Overview](data-clone/overview.md)
+  - [Connection Manager](data-clone/connection-manager.md)
+  - [Sync Modes](data-clone/sync-modes.md)
+  - [Configuration](data-clone/configuration.md)
+  - [Docker Deployment](data-clone/docker.md)
+
+- CLI Reference
+
+  - [CLI Overview](cli/overview.md)
+  - [csvcheck](cli/csvcheck.md)
+  - [csvtransform](cli/csvtransform.md)
+  - [tsvtransform](cli/tsvtransform.md)
+  - [jsonarraytransform](cli/jsonarraytransform.md)
+  - [comprehensionintersect](cli/comprehensionintersect.md)
+  - [comprehensionarray](cli/comprehensionarray.md)
+  - [objectarraytocsv](cli/objectarraytocsv.md)
+  - [load_comprehension](cli/load-comprehension.md)
+  - [data-clone](cli/data-clone.md)
+  - [serve](cli/serve.md)
+
+- REST API Reference
+
+  - [REST Overview](rest/overview.md)
+  - [Status](rest/status.md)
+  - [CSV Operations](rest/csv.md)
+  - [TSV Operations](rest/tsv.md)
+  - [JSON Array Operations](rest/json-array.md)
+  - [Comprehension Operations](rest/comprehension.md)
+  - [Entity Generation](rest/entity-generation.md)
+  - [Comprehension Push](rest/comprehension-push.md)
+
+- API Reference
 
-  - [CLI Reference](cli-reference.md)
-  - [REST API Reference](rest-api-reference.md)
-  - [Programmatic API](programmatic-api.md)
-  - [Integration Adapter](integration-adapter.md)
+  - [ConnectionManager](api/connection-manager.md)
+  - [CloneRestClient](api/clone-rest-client.md)
+  - [Sync](api/sync.md)
+  - [SyncEntityInitial](api/sync-entity-initial.md)
+  - [SyncEntityOngoing](api/sync-entity-ongoing.md)
+  - [Operation](api/operation.md)
+  - [IntegrationAdapter](api/integration-adapter.md)
+  - [GUIDMap](api/guid-map.md)
+  - [TabularCheck](api/tabular-check.md)
+  - [TabularTransform](api/tabular-transform.md)
 
 - Examples
 

package/docs/_topbar.md

@@ -1,7 +1,9 @@
 # Meadow Integration
 
 - [Overview](README.md)
+- [Architecture](architecture.md)
 - [CLI Reference](cli-reference.md)
 - [API Reference](programmatic-api.md)
+- [Data Clone](data-clone/overview.md)
 - [Examples](examples-walkthrough.md)
 - [GitHub](https://github.com/stevenvelozo/meadow-integration)