@loxtep/sdk 0.7.22 → 0.7.26

package/docs/sdk-first-ingest.md

@@ -22,20 +22,34 @@ You should see `project_id` and `instance_id` from `attach`.
 
 ---
 
-## Phase 2 — Create your data product
+## Phase 2 — Create your data product (local package)
 
 Pick a name your app will use (example: `app-events`):
 
 ```bash
+pnpm exec loxtep connectors list --type sdk   # optional: see existing SDK connectors
 pnpm exec loxtep ingest provision --name app-events
+pnpm exec loxtep lint
 ```
 
-That is all you need for SDK writes — one command, one name.
+`ingest provision` reuses an existing SDK connector when one exists (or creates one),
+then writes a **local** workflow package under `connectors/` and `workflows/<id>/`.
+It does **not** deploy by default. Validate with `loxtep lint`, then publish:
+
+```bash
+pnpm exec loxtep deploy --dry-run   # lint only
+pnpm exec loxtep deploy             # lint + compile/deploy modules
+# or: pnpm exec loxtep ingest provision --name app-events --deploy
+```
+
+That local package is what you iterate on; deploy pushes it to your instance.
 
 ---
 
 ## Phase 3 — Confirm it exists
 
+After deploy:
+
 ```bash
 pnpm exec loxtep data-products list
 ```
@@ -90,6 +104,8 @@ pnpm exec loxtep login
 pnpm exec loxtep init
 pnpm exec loxtep attach --instance <instance-id>
 pnpm exec loxtep ingest provision --name app-events
+pnpm exec loxtep lint
+pnpm exec loxtep deploy
 pnpm exec loxtep data-products list
 ```
 
@@ -102,7 +118,7 @@ Then `get_writer('app-events')` in your app.
 | What you see | What to do |
 | ------------ | ---------- |
 | Data product not found | Run `data-products list`. Check the name matches `get_writer('…')` exactly. |
-| Not ready to accept writes | Run `ingest provision` again for that name. Make sure you ran `attach` first. |
+| Not ready to accept writes | Run `ingest provision`, then `deploy` (or `ingest provision --deploy`). Make sure you ran `attach` first. |
 | Multiple products with the same name | Pass the data product id to `get_writer`, or ensure `attach` points at the right instance. |
 | No domain available | Create a domain in the Loxtep UI, or pass `--domain-id` to `ingest provision`. |
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loxtep/sdk",
-  "version": "0.7.22",
+  "version": "0.7.26",
   "description": "Loxtep SDK for Node.js — the Enterprise Context Layer: data products, workflows, projects, queues, and machine-usable context for AI",
   "author": "Loxtep <engineering@loxtep.io>",
   "homepage": "https://loxtep.io",
@@ -33,13 +33,15 @@
   "files": [
     "dist",
     "docs",
+    "schemas",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && mkdir -p dist/lib/entity-json-schemas && cp src/lib/entity-json-schemas/ajv-loader.cjs dist/lib/entity-json-schemas/ajv-loader.cjs",
     "prepublishOnly": "npm run build",
     "test": "jest --config jest.config.cjs",
     "lint": "eslint src --ext .ts",
+    "sync:entity-schemas": "node scripts/sync-entity-schemas.mjs",
     "generate:api-types": "tsx scripts/generate-api-types.ts",
     "docs": "typedoc"
   },
@@ -82,6 +84,8 @@
     "@smithy/protocol-http": "^5.4.3",
     "@smithy/signature-v4": "^5.4.3",
     "@smithy/types": "^4.14.2",
+    "ajv": "^8.20.0",
+    "ajv-formats": "^3.0.1",
     "js-yaml": "^4.3.0",
     "leo-sdk": "7.1.21",
     "zod": "^4.4.3"

package/schemas/entity-json-schemas/SCHEMA_SYNC.md

@@ -0,0 +1,18 @@
+# Entity JSON schemas (vendored)
+
+These JSON Schema draft-07 files are a copy of the Loxtep platform schemas used
+for customer workspace entity validation.
+
+**Source (canonical):**
+`loxtep/platform-backend/_core/src/customer-workspace/entity-json-schemas/`
+
+**Sync command** (from `nodejs/`):
+
+```bash
+node scripts/sync-entity-schemas.mjs /path/to/loxtep
+```
+
+When platform schemas change, re-run the sync script and note the date/source
+commit in the SDK CHANGELOG.
+
+**Last synced:** 2026-07-24 (from local loxtep checkout)

package/schemas/entity-json-schemas/connection.json

@@ -0,0 +1,310 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/connection.json",
+  "title": "Connection Entity",
+  "description": "JSON Schema for Connection entity in customer workspace. Connections are configured instances of connectors with credentials and settings.",
+  "type": "object",
+  "required": [
+    "connection_id",
+    "organization_id",
+    "project_id",
+    "key",
+    "name",
+    "type",
+    "status"
+  ],
+  "properties": {
+    "connection_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the connection"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this connection"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this connection"
+    },
+    "workflow_id": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "Workflow this connection belongs to; null for project-level/draft connections"
+    },
+    "connector_id": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "Reference to the connector definition this connection uses; null for project-level/draft connections"
+    },
+    "sync_entity": {
+      "type": ["string", "null"],
+      "pattern": "^[a-z0-9-_]+$",
+      "description": "For API poll connectors: which entity (e.g. orders, customers) this connection syncs. Must match connector supported_entities when present."
+    },
+    "key": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "pattern": "^[a-z0-9-_]+$",
+      "description": "Unique key for the connection within the organization"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Display name for the connection"
+    },
+    "description": {
+      "type": ["string", "null"],
+      "maxLength": 1000,
+      "description": "Connection description"
+    },
+    "type": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Connection type: database, api, webhook, file, or connector type (e.g. spotify, shopify)"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["active", "inactive", "error"],
+      "description": "Connection status"
+    },
+    "credential_fields": {
+      "type": "object",
+      "description": "Encrypted credential fields for the connection (structure depends on connector auth_type)",
+      "properties": {
+        "api_key": {
+          "type": ["string", "null"],
+          "description": "API key (encrypted)"
+        },
+        "api_secret": {
+          "type": ["string", "null"],
+          "description": "API secret (encrypted)"
+        },
+        "oauth_access_token": {
+          "type": ["string", "null"],
+          "description": "OAuth access token (encrypted)"
+        },
+        "oauth_refresh_token": {
+          "type": ["string", "null"],
+          "description": "OAuth refresh token (encrypted)"
+        },
+        "oauth_expires_at": {
+          "type": ["string", "null"],
+          "format": "date-time",
+          "description": "OAuth token expiration time"
+        },
+        "username": {
+          "type": ["string", "null"],
+          "description": "Username for basic auth (encrypted)"
+        },
+        "password": {
+          "type": ["string", "null"],
+          "description": "Password for basic auth (encrypted)"
+        },
+        "bearer_token": {
+          "type": ["string", "null"],
+          "description": "Bearer token (encrypted)"
+        },
+        "custom_credentials": {
+          "type": ["object", "null"],
+          "additionalProperties": {
+            "type": "string"
+          },
+          "description": "Custom credential fields (encrypted)"
+        }
+      },
+      "additionalProperties": false
+    },
+    "credential_parameter_store_refs": {
+      "type": "object",
+      "description": "SSM Parameter Store paths for OAuth/credentials (keyed by credential name)",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "configuration": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Connection configuration (non-sensitive settings)"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the connection"
+    },
+    "verified": {
+      "type": "boolean",
+      "default": false,
+      "description": "Whether the connection has been verified/tested"
+    },
+    "draft": {
+      "type": "boolean",
+      "default": true,
+      "description": "Whether this connection is still in draft state"
+    },
+    "last_tested": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connection was last tested"
+    },
+    "created_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who created the connection"
+    },
+    "updated_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who last updated the connection"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connection was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connection was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connection was deleted (soft delete)"
+    },
+    "ingestion_config": {
+      "type": ["object", "null"],
+      "description": "Structured ingestion settings (schedule, batch, polling). Extensible per connection type.",
+      "properties": {
+        "schedule": {
+          "type": ["string", "null"],
+          "description": "Cron expression (LeoCron `time`) or platform-supported schedule string for this connection's registered bot. Distinct from `polling_interval_ms`, which is a millisecond gap for application-level throttling between requests—not the Leo schedule field."
+        },
+        "runtime_memory_mb": {
+          "description": "Shared runtimes tier for trigger registration (256–2048). Defaults to 256 when unset. LeoCron `lambdaName`: `w-{org4}-{inst4}-bot-connectors-{mb}-rtime`; bot id: `w-{org4}-{inst4}-{workflow4}-bot-connectors-{mb}-{container_id}`. Instance `connection_details.runtimes_stack_identifier` may override the org/instance middle segment (CDK `microserviceIdentifier` = `w-{that middle}`).",
+          "oneOf": [
+            { "type": "null" },
+            { "type": "integer", "enum": [256, 512, 1024, 2048] }
+          ]
+        },
+        "fetch_action_key": {
+          "type": ["string", "null"],
+          "description": "Declarative connector package action key used for polling fetches (e.g. shopify_get_orders). Often matches sync_plan entity fetch_action_key."
+        },
+        "batch_size": {
+          "type": ["integer", "null"],
+          "description": "Batch size for polling (default 100)"
+        },
+        "polling_interval_ms": {
+          "type": ["integer", "null"],
+          "description": "Polling interval in milliseconds"
+        },
+        "max_retries": {
+          "type": ["integer", "null"],
+          "description": "Max retries on failure"
+        },
+        "timeout_ms": {
+          "type": ["integer", "null"],
+          "description": "Request/timeout in milliseconds"
+        }
+      },
+      "additionalProperties": true
+    },
+    "runtime_status": {
+      "type": ["string", "null"],
+      "enum": ["not_deployed", "deploying", "running", "paused", "error", null],
+      "description": "Deployment state of the connection's bot (written at deploy time)"
+    },
+    "auth_type": {
+      "type": ["string", "null"],
+      "enum": [
+        "oauth2",
+        "api_key",
+        "webhook",
+        "data_product_rbac",
+        "none",
+        null
+      ],
+      "description": "Authentication type for the connection (optional, backwards compat)."
+    },
+    "ingestion_type": {
+      "type": ["string", "null"],
+      "enum": [
+        "polling",
+        "webhook",
+        "data_product",
+        "database_cdc",
+        "file",
+        "api_sdk",
+        "stream",
+        "other",
+        null
+      ],
+      "description": "How data is ingested (optional)."
+    },
+    "export_type": {
+      "type": ["string", "null"],
+      "enum": ["file", "database", "stream", "api_sdk", "other", null],
+      "description": "How data is exported (optional)."
+    },
+    "source_schema": {
+      "type": ["object", "null"],
+      "description": "Source schema reference (optional)."
+    },
+    "destination_schema": {
+      "type": ["object", "null"],
+      "description": "Destination schema reference (optional)."
+    },
+    "retry_policy": {
+      "type": ["object", "null"],
+      "description": "Retry policy (max_retries, backoff, etc.).",
+      "properties": {
+        "max_retries": { "type": ["integer", "null"] },
+        "backoff_ms": { "type": ["integer", "null"] }
+      },
+      "additionalProperties": true
+    },
+    "timeout": {
+      "type": ["integer", "null"],
+      "description": "Timeout in milliseconds (optional)."
+    },
+    "concurrency": {
+      "type": ["integer", "null"],
+      "minimum": 1,
+      "description": "Concurrency limit (optional)."
+    },
+    "batch_size": {
+      "type": ["integer", "null"],
+      "minimum": 1,
+      "description": "Batch size for sync (optional)."
+    },
+    "connector_mode": {
+      "type": ["string", "null"],
+      "enum": ["inbound", "outbound", "bidirectional", null],
+      "description": "For enrichment workflows: mid-flow connection must be 'bidirectional'. For ingestion head use 'inbound'; for consumption tail use 'outbound'. Must match connector supported_directions."
+    }
+  },
+  "upstream_entity_id": {
+    "type": ["string", "null"],
+    "format": "uuid",
+    "description": "Upstream entity in the flow graph (e.g. data-product in consumption flows). Null for source connections."
+  },
+  "upstream_entity_type": {
+    "type": ["string", "null"],
+    "enum": [
+      "connections",
+      "transformations",
+      "validations",
+      "data-products",
+      null
+    ],
+    "description": "Entity type of the upstream entity."
+  },
+  "additionalProperties": false
+}

package/schemas/entity-json-schemas/connector.json

@@ -0,0 +1,188 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/connector.json",
+  "title": "Connector Entity",
+  "description": "JSON Schema for Connector entity in customer workspace. Connectors define reusable integration templates (e.g., Shopify, Stripe, HubSpot).",
+  "type": "object",
+  "required": [
+    "connector_id",
+    "organization_id",
+    "connector_type",
+    "name",
+    "description",
+    "category",
+    "auth_type",
+    "version",
+    "created_at",
+    "updated_at"
+  ],
+  "properties": {
+    "connector_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the connector"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this connector definition"
+    },
+    "connector_type": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 100,
+      "pattern": "^[a-z0-9-_]+$",
+      "description": "Connector type identifier (e.g., 'shopify', 'stripe', 'hubspot')"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Display name for the connector"
+    },
+    "description": {
+      "type": "string",
+      "maxLength": 1000,
+      "description": "Connector description"
+    },
+    "category": {
+      "type": "string",
+      "enum": [
+        "e_commerce",
+        "crm",
+        "marketing",
+        "finance",
+        "database",
+        "file_storage",
+        "analytics",
+        "communication",
+        "productivity",
+        "custom"
+      ],
+      "description": "Connector category"
+    },
+    "icon_url": {
+      "type": ["string", "null"],
+      "format": "uri",
+      "description": "URL to connector icon"
+    },
+    "documentation_url": {
+      "type": ["string", "null"],
+      "format": "uri",
+      "description": "URL to connector documentation"
+    },
+    "auth_type": {
+      "type": "string",
+      "enum": ["oauth2", "api_key", "basic", "bearer", "custom", "none"],
+      "description": "Authentication type required for this connector"
+    },
+    "oauth_config": {
+      "type": ["object", "null"],
+      "description": "OAuth2 configuration (required if auth_type is oauth2)",
+      "properties": {
+        "authorization_url": {
+          "type": "string",
+          "format": "uri"
+        },
+        "token_url": {
+          "type": "string",
+          "format": "uri"
+        },
+        "scopes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 0
+        },
+        "response_type": {
+          "type": ["string", "null"]
+        },
+        "grant_type": {
+          "type": ["string", "null"]
+        },
+        "additional_params": {
+          "type": ["object", "null"],
+          "additionalProperties": {
+            "type": "string"
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "supported_directions": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": ["inbound", "outbound", "bidirectional"]
+      },
+      "minItems": 1,
+      "description": "Data sync directions supported by this connector"
+    },
+    "supported_entities": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      },
+      "minItems": 0,
+      "description": "Entity types supported by this connector (e.g., ['orders', 'customers', 'products'])"
+    },
+    "supports_webhooks": {
+      "type": "boolean",
+      "default": false,
+      "description": "Whether this connector supports webhook-based sync"
+    },
+    "supports_polling": {
+      "type": "boolean",
+      "default": false,
+      "description": "Whether this connector supports polling-based sync"
+    },
+    "required_scopes": {
+      "type": ["array", "null"],
+      "items": {
+        "type": "string"
+      },
+      "description": "OAuth scopes required for this connector"
+    },
+    "required_permissions": {
+      "type": ["array", "null"],
+      "items": {
+        "type": "string"
+      },
+      "description": "Permissions required for this connector"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+$",
+      "description": "Connector version (semantic versioning)"
+    },
+    "is_beta": {
+      "type": "boolean",
+      "default": false,
+      "description": "Whether this connector is in beta"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the connector"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connector was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connector was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when connector was deleted (soft delete)"
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/entity-json-schemas/container.json

@@ -0,0 +1,71 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/container.json",
+  "title": "Container Entity",
+  "description": "JSON Schema for Container entity in customer workspace. A container represents one logical node that maps to one or more bots/queues at runtime. It wraps another project entity (connection, transformation, validation, data-product, export).",
+  "type": "object",
+  "required": [
+    "container_id",
+    "organization_id",
+    "project_id",
+    "workflow_id",
+    "wrapped_entity_type",
+    "wrapped_entity_id"
+  ],
+  "properties": {
+    "container_id": {
+      "type": "string",
+      "pattern": "^[a-z0-9-]+$",
+      "minLength": 1,
+      "maxLength": 64,
+      "description": "Unique identifier for the container (e.g. conn-a1b2c3d4, xfm-e5f6g7h8)"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this container"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this container"
+    },
+    "workflow_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Workflow this container belongs to"
+    },
+    "wrapped_entity_type": {
+      "type": "string",
+      "enum": [
+        "connections",
+        "transformations",
+        "validations",
+        "data-products",
+        "exports",
+        "workflows"
+      ],
+      "description": "Entity type of the wrapped project entity"
+    },
+    "wrapped_entity_id": {
+      "type": "string",
+      "description": "Entity ID of the wrapped project entity (e.g. connection_id, transformation_id)"
+    },
+    "runtime": {
+      "type": "object",
+      "description": "Optional runtime slice written at deploy time; when present, project is self-describing",
+      "properties": {
+        "bot_ids": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "rstreams bot IDs registered for this container"
+        },
+        "queue_ids": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "rstreams queue names for this container"
+        }
+      }
+    }
+  }
+}