@loxtep/sdk 0.7.21 → 0.7.24

package/schemas/entity-json-schemas/validation.json

@@ -0,0 +1,152 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/validation.json",
+  "title": "Validation Entity",
+  "description": "JSON Schema for Validation entity in customer workspace. Validations are rules that assert data quality and reject invalid events within a workflow flow. Each validation references its workflow via workflow_id and the upstream entity it applies to via upstream_entity_id.",
+  "type": "object",
+  "required": [
+    "validation_id",
+    "organization_id",
+    "project_id",
+    "workflow_id",
+    "name",
+    "validation_type",
+    "rules",
+    "severity",
+    "created_at",
+    "updated_at"
+  ],
+  "properties": {
+    "validation_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the validation"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this validation"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this validation"
+    },
+    "workflow_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Workflow this validation belongs to"
+    },
+    "upstream_entity_id": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "Reference to the upstream entity this validation applies to (connection, transformation, or data product)"
+    },
+    "upstream_entity_type": {
+      "type": ["string", "null"],
+      "enum": ["connections", "transformations", "data-products", null],
+      "description": "Entity type of the upstream entity"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Validation name"
+    },
+    "description": {
+      "type": ["string", "null"],
+      "maxLength": 1000,
+      "description": "Validation description"
+    },
+    "validation_type": {
+      "type": "string",
+      "enum": [
+        "schema",
+        "business_rule",
+        "data_quality",
+        "pii_check",
+        "completeness",
+        "format"
+      ],
+      "description": "Type of validation"
+    },
+    "schema_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Org domain schema reference; deploy resolves latest active version into rules.schema when rules.schema is empty"
+    },
+    "schema_version_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Pinned org domain schema version; deploy resolves definition into rules.schema when rules.schema is empty"
+    },
+    "rules": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Validation rules (structure depends on validation_type). For validation_type schema, rules.schema holds the JSON Schema; rules.schema_id may reference org registry instead.",
+      "examples": [
+        {
+          "schema": {
+            "type": "object",
+            "description": "JSON Schema for validation"
+          }
+        },
+        {
+          "condition": "field > 100",
+          "message": "Field must be greater than 100"
+        },
+        {
+          "required_fields": ["email", "name"],
+          "completeness_threshold": 0.95
+        }
+      ]
+    },
+    "severity": {
+      "type": "string",
+      "enum": ["error", "warning", "info"],
+      "description": "Severity level when validation fails"
+    },
+    "error_message": {
+      "type": ["string", "null"],
+      "maxLength": 500,
+      "description": "Custom error message when validation fails"
+    },
+    "order": {
+      "type": ["integer", "null"],
+      "minimum": 0,
+      "description": "Default execution order when used in workflows (lower = earlier)"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the validation"
+    },
+    "created_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who created the validation"
+    },
+    "updated_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who last updated the validation"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when validation was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when validation was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when validation was deleted (soft delete)"
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/entity-json-schemas/workflow-graph.json

@@ -0,0 +1,116 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/workflow-graph.json",
+  "title": "Workflow Graph",
+  "description": "Materialized read-only graph view for a workflow (flow). Built from per-entity S3 files; not a writable entity type.",
+  "type": "object",
+  "required": [
+    "workflow_id",
+    "organization_id",
+    "project_id",
+    "version",
+    "built_at",
+    "workflow",
+    "nodes",
+    "edges",
+    "connectors",
+    "deployment",
+    "metadata"
+  ],
+  "properties": {
+    "workflow_id": { "type": "string", "format": "uuid" },
+    "organization_id": { "type": "string", "format": "uuid" },
+    "project_id": { "type": "string", "format": "uuid" },
+    "version": { "type": "integer", "minimum": 1 },
+    "built_at": { "type": "string", "format": "date-time" },
+    "applied_template_digest": { "type": ["string", "null"] },
+    "workflow": {
+      "type": "object",
+      "description": "Full workflow.json entity snapshot"
+    },
+    "nodes": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["entity_type", "entity_id", "data"],
+        "properties": {
+          "entity_type": { "type": "string" },
+          "entity_id": { "type": "string", "format": "uuid" },
+          "upstream_entity_id": { "type": ["string", "null"] },
+          "deployment_status": {
+            "type": "string",
+            "enum": ["undeployed", "deployed", "stale"]
+          },
+          "container_id": { "type": ["string", "null"] },
+          "data": { "type": "object" }
+        }
+      }
+    },
+    "edges": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["from", "to", "type"],
+        "properties": {
+          "from": { "type": "string" },
+          "to": { "type": "string" },
+          "type": { "type": "string", "enum": ["upstream"] }
+        }
+      }
+    },
+    "connectors": {
+      "type": "object",
+      "additionalProperties": { "type": "object" }
+    },
+    "deployment": {
+      "type": "object",
+      "required": [
+        "deployment_id",
+        "deployed_at",
+        "version_id",
+        "microservice_id",
+        "runtime_mapping"
+      ],
+      "properties": {
+        "deployment_id": { "type": ["string", "null"], "format": "uuid" },
+        "deployed_at": { "type": ["string", "null"], "format": "date-time" },
+        "version_id": { "type": ["string", "null"], "format": "uuid" },
+        "microservice_id": { "type": ["string", "null"] },
+        "runtime_mapping": {
+          "type": "object",
+          "properties": {
+            "containers": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "container_id": { "type": "string" },
+                  "entity_id": { "type": "string" },
+                  "entity_type": { "type": "string" },
+                  "bot_ids": { "type": "array", "items": { "type": "string" } },
+                  "queue_ids": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  }
+                }
+              }
+            },
+            "all_bot_ids": { "type": "array", "items": { "type": "string" } },
+            "all_queue_ids": { "type": "array", "items": { "type": "string" } }
+          }
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "node_count": { "type": "integer" },
+        "entity_type_counts": {
+          "type": "object",
+          "additionalProperties": { "type": "integer" }
+        },
+        "requires_redeployment": { "type": "boolean" }
+      }
+    }
+  }
+}

package/schemas/entity-json-schemas/workflow.json

@@ -0,0 +1,193 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/workflow.json",
+  "title": "Workflow Entity",
+  "description": "JSON Schema for the root workflow entity in customer workspace. A workflow is a flow. Flow nodes (connection, transformation, validation, data product) reference the workflow via workflow_id and point upstream via upstream_entity_id; the workflow does not store node IDs. template_id identifies the flow starter template used to create this flow (e.g. blank, webhook-ingestion), distinct from connector catalog templates.",
+  "type": "object",
+  "required": [
+    "workflow_id",
+    "organization_id",
+    "project_id",
+    "name",
+    "template_id",
+    "workflow_type",
+    "domain_id",
+    "status",
+    "created_at",
+    "updated_at"
+  ],
+  "properties": {
+    "workflow_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the workflow"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this workflow"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this workflow"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Workflow display name"
+    },
+    "description": {
+      "type": ["string", "null"],
+      "maxLength": 1000,
+      "description": "Workflow description"
+    },
+    "template_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Workflow starter template identifier used when this flow was created (e.g. blank, webhook-ingestion). Distinct from connector catalog templates."
+    },
+    "workflow_type": {
+      "type": "string",
+      "enum": ["ingestion", "enrichment", "consumption"],
+      "description": "Type of flow: ingestion (ingest from external sources), enrichment (transform/join data), consumption (expose or consume data products)."
+    },
+    "domain_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Domain this workflow belongs to (required)."
+    },
+    "segments": {
+      "type": "array",
+      "default": [],
+      "description": "Visual groupings of nodes in the graph editor.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "segment_id": {
+            "type": "string",
+            "description": "Unique identifier for the segment"
+          },
+          "name": { "type": "string", "description": "Display name" },
+          "description": {
+            "type": ["string", "null"],
+            "description": "Optional description"
+          },
+          "node_ids": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Entity IDs of nodes in this segment"
+          },
+          "color": {
+            "type": ["string", "null"],
+            "description": "Optional color for UI"
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "trigger": {
+      "type": ["object", "null"],
+      "description": "Workflow trigger configuration (event, schedule, or connector).",
+      "properties": {
+        "event": {
+          "type": ["string", "null"],
+          "description": "Event-based trigger (e.g. webhook, queue)"
+        },
+        "schedule": {
+          "type": ["string", "null"],
+          "description": "Cron or interval for scheduled execution"
+        },
+        "connector": {
+          "type": ["string", "null"],
+          "format": "uuid",
+          "description": "Connector ID for connector-driven triggers"
+        }
+      },
+      "additionalProperties": false
+    },
+    "configuration": {
+      "type": "object",
+      "description": "Workflow configuration (schedule, batch_size, etc.). Flow nodes are discovered by workflow_id and upstream_entity_id; the workflow does not store connection_id or transformation_ids.",
+      "properties": {
+        "schedule": {
+          "type": ["string", "null"],
+          "description": "Cron expression for scheduled execution (optional)"
+        },
+        "batch_size": {
+          "type": ["integer", "null"],
+          "minimum": 1,
+          "description": "Number of events to process per batch"
+        }
+      },
+      "additionalProperties": false
+    },
+    "deployment": {
+      "type": ["object", "null"],
+      "description": "Deployment information (set when workflow is deployed)",
+      "properties": {
+        "instance_id": {
+          "type": "string",
+          "format": "uuid"
+        },
+        "lambda_functions": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "queues": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "status": {
+      "type": "string",
+      "enum": ["active", "paused", "error", "pending", "inactive"],
+      "description": "Workflow status"
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Workflow metrics (events processed, error rate, etc.)"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the workflow"
+    },
+    "created_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who created the workflow"
+    },
+    "updated_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who last updated the workflow"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when workflow was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when workflow was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when workflow was deleted (soft delete)"
+    }
+  },
+  "additionalProperties": false
+}

package/docs/examples/generate-ingest-bundle.mjs

@@ -1,133 +0,0 @@
-#!/usr/bin/env node
-/**
- * Greenfield SDK ingest — one-shot provision via CLI (preferred) or bundle JSON for manual save.
- *
- * Run from your Loxtep workspace root (after login, init, attach):
- *
- *   pnpm exec loxtep ingest provision --name app-events
- *
- * Or generate bundle JSON only:
- *
- *   node node_modules/@loxtep/sdk/docs/examples/generate-ingest-bundle.mjs
- *   pnpm exec loxtep bundle save --dry-run
- *   pnpm exec loxtep bundle save
- */
-
-import { LoxtepClient, buildSdkIngestBundle } from '@loxtep/sdk';
-import { randomUUID } from 'node:crypto';
-import { writeFileSync, mkdirSync } from 'node:fs';
-import { join } from 'node:path';
-
-const DATA_PRODUCT_NAME = process.env.LOXTEP_DATA_PRODUCT_NAME ?? 'app-events';
-const WORKFLOW_NAME = process.env.LOXTEP_WORKFLOW_NAME ?? 'SDK App Events Ingest';
-const AUTO_SAVE = process.env.LOXTEP_AUTO_SAVE === '1';
-
-async function main() {
-  const client = await LoxtepClient.fromWorkspace();
-
-  const user = await client.session.get_current_user();
-  const organizationId = user.organization_id ?? client.organization_id;
-  const projectId = client.project_id;
-  const instanceId = client.instance_id;
-
-  if (!organizationId || !projectId) {
-    console.error(
-      'Missing organization_id or project_id. Run from a workspace after `loxtep init`.'
-    );
-    process.exit(1);
-  }
-  if (!instanceId) {
-    console.error('Missing instance_id. Run `loxtep attach --instance <id>` first.');
-    process.exit(1);
-  }
-
-  let domainId = process.env.LOXTEP_DOMAIN_ID;
-  if (!domainId) {
-    const domains = await client.define.domains.list({ page_size: 20 });
-    const items = domains.items ?? [];
-    if (items.length === 0) {
-      console.error(
-        'No domains found. Create one in the Web UI (Governance → Domains) or set LOXTEP_DOMAIN_ID.'
-      );
-      process.exit(1);
-    }
-    domainId = items[0].domain_id;
-    console.error(`Using domain: ${items[0].name} (${domainId})`);
-  }
-
-  console.error('Creating SDK connector…');
-  const connector = await client.connect.connectors.create({
-    connector_type: 'sdk',
-    metadata: { name: 'SDK Connector', created_by: 'generate-ingest-bundle' },
-  });
-
-  const bundle = buildSdkIngestBundle({
-    organization_id: organizationId,
-    project_id: projectId,
-    domain_id: domainId,
-    connector_id: connector.connector_id,
-    data_product_name: DATA_PRODUCT_NAME,
-    workflow_name: WORKFLOW_NAME,
-    user_id: user.user_id,
-    workflow_id: randomUUID(),
-    connection_id: randomUUID(),
-    data_product_id: randomUUID(),
-  });
-
-  mkdirSync('.loxtep', { recursive: true });
-  const outPath = join('.loxtep', 'sdk-ingest-bundle.json');
-  writeFileSync(outPath, JSON.stringify({ project_id: projectId, files: bundle.files }, null, 2));
-
-  console.log(
-    JSON.stringify(
-      {
-        connector_id: connector.connector_id,
-        workflow_id: bundle.workflow_id,
-        data_product_id: bundle.data_product_id,
-        data_product_name: bundle.data_product_name,
-        bundle_path: outPath,
-      },
-      null,
-      2
-    )
-  );
-
-  if (AUTO_SAVE) {
-    console.error('Saving workflow bundle (LOXTEP_AUTO_SAVE=1)…');
-    const saveResult = await client.build.workflows.save_workflow_bundle(projectId, {
-      files: bundle.files,
-      dry_run: false,
-    });
-    console.error('Deploying…');
-    const deployResult = await client.build.workflows.deploy({
-      project_id: projectId,
-      instance_id: instanceId,
-    });
-    console.log(JSON.stringify({ save: saveResult, deploy: deployResult }, null, 2));
-    console.error(`
-Provisioned. Write events:
-  LOXTEP_DATA_PRODUCT_NAME=${DATA_PRODUCT_NAME} node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
-`);
-    return;
-  }
-
-  console.error(`
-Next steps (no MCP required)
-────────────────────────────
-Preferred — one command:
-  pnpm exec loxtep ingest provision --name ${DATA_PRODUCT_NAME}
-
-Or save the generated bundle:
-  pnpm exec loxtep bundle save --dry-run --file ${outPath}
-  pnpm exec loxtep bundle save --file ${outPath}
-  pnpm exec loxtep workflows deploy --project-id ${projectId} --instance-id ${instanceId}
-
-Write events:
-  LOXTEP_DATA_PRODUCT_NAME=${DATA_PRODUCT_NAME} node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
-`);
-}
-
-main().catch(err => {
-  console.error(err instanceof Error ? err.message : err);
-  process.exit(1);
-});

package/docs/examples/write-events.mjs

@@ -1,54 +0,0 @@
-#!/usr/bin/env node
-/**
- * Write sample events to a deployed source data product via get_writer.
- *
- * Prerequisites: login, init, attach, SDK ingest workflow deployed (see sdk-first-ingest.md).
- *
- *   node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
- *
- * Env:
- *   LOXTEP_DATA_PRODUCT_NAME  default: app-events
- */
-
-import { LoxtepClient } from '@loxtep/sdk';
-
-const DATA_PRODUCT_NAME = process.env.LOXTEP_DATA_PRODUCT_NAME ?? 'app-events';
-
-async function main() {
-  const client = await LoxtepClient.fromWorkspace();
-
-  console.error(`Opening writer for data product "${DATA_PRODUCT_NAME}"…`);
-  const writer = await client.get_writer(DATA_PRODUCT_NAME);
-
-  const events = [
-    {
-      event_id: `evt_${Date.now()}_1`,
-      occurred_at: new Date().toISOString(),
-      payload: { source: 'write-events.mjs', action: 'example_write', index: 1 },
-    },
-    {
-      event_id: `evt_${Date.now()}_2`,
-      occurred_at: new Date().toISOString(),
-      payload: { source: 'write-events.mjs', action: 'example_write', index: 2 },
-    },
-  ];
-
-  for (const event of events) {
-    writer.write(event);
-    console.log(JSON.stringify({ written: event.event_id }));
-  }
-
-  await writer.close();
-  console.error('Writer closed — events flushed.');
-}
-
-main().catch(err => {
-  const message = err instanceof Error ? err.message : String(err);
-  console.error(message);
-  if (/not deployed/i.test(message)) {
-    console.error(
-      '\nDeploy an SDK ingestion workflow first. See nodejs/docs/sdk-first-ingest.md'
-    );
-  }
-  process.exit(1);
-});