@loxtep/sdk 0.7.15 → 0.7.18

package/docs/examples/generate-ingest-bundle.mjs

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+/**
+ * Greenfield SDK ingest — one-shot provision via CLI (preferred) or bundle JSON for manual save.
+ *
+ * Run from your Loxtep workspace root (after login, init, attach):
+ *
+ *   pnpm exec loxtep ingest provision --name app-events
+ *
+ * Or generate bundle JSON only:
+ *
+ *   node node_modules/@loxtep/sdk/docs/examples/generate-ingest-bundle.mjs
+ *   pnpm exec loxtep bundle save --dry-run
+ *   pnpm exec loxtep bundle save
+ */
+
+import { LoxtepClient, buildSdkIngestBundle } from '@loxtep/sdk';
+import { randomUUID } from 'node:crypto';
+import { writeFileSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+const DATA_PRODUCT_NAME = process.env.LOXTEP_DATA_PRODUCT_NAME ?? 'app-events';
+const WORKFLOW_NAME = process.env.LOXTEP_WORKFLOW_NAME ?? 'SDK App Events Ingest';
+const AUTO_SAVE = process.env.LOXTEP_AUTO_SAVE === '1';
+
+async function main() {
+  const client = await LoxtepClient.fromWorkspace();
+
+  const user = await client.session.get_current_user();
+  const organizationId = user.organization_id ?? client.organization_id;
+  const projectId = client.project_id;
+  const instanceId = client.instance_id;
+
+  if (!organizationId || !projectId) {
+    console.error(
+      'Missing organization_id or project_id. Run from a workspace after `loxtep init`.'
+    );
+    process.exit(1);
+  }
+  if (!instanceId) {
+    console.error('Missing instance_id. Run `loxtep attach --instance <id>` first.');
+    process.exit(1);
+  }
+
+  let domainId = process.env.LOXTEP_DOMAIN_ID;
+  if (!domainId) {
+    const domains = await client.define.domains.list({ page_size: 20 });
+    const items = domains.items ?? [];
+    if (items.length === 0) {
+      console.error(
+        'No domains found. Create one in the Web UI (Governance → Domains) or set LOXTEP_DOMAIN_ID.'
+      );
+      process.exit(1);
+    }
+    domainId = items[0].domain_id;
+    console.error(`Using domain: ${items[0].name} (${domainId})`);
+  }
+
+  console.error('Creating SDK connector…');
+  const connector = await client.connect.connectors.create({
+    connector_type: 'sdk',
+    metadata: { name: 'SDK Connector', created_by: 'generate-ingest-bundle' },
+  });
+
+  const bundle = buildSdkIngestBundle({
+    organization_id: organizationId,
+    project_id: projectId,
+    domain_id: domainId,
+    connector_id: connector.connector_id,
+    data_product_name: DATA_PRODUCT_NAME,
+    workflow_name: WORKFLOW_NAME,
+    user_id: user.user_id,
+    workflow_id: randomUUID(),
+    connection_id: randomUUID(),
+    data_product_id: randomUUID(),
+  });
+
+  mkdirSync('.loxtep', { recursive: true });
+  const outPath = join('.loxtep', 'sdk-ingest-bundle.json');
+  writeFileSync(outPath, JSON.stringify({ project_id: projectId, files: bundle.files }, null, 2));
+
+  console.log(
+    JSON.stringify(
+      {
+        connector_id: connector.connector_id,
+        workflow_id: bundle.workflow_id,
+        data_product_id: bundle.data_product_id,
+        data_product_name: bundle.data_product_name,
+        bundle_path: outPath,
+      },
+      null,
+      2
+    )
+  );
+
+  if (AUTO_SAVE) {
+    console.error('Saving workflow bundle (LOXTEP_AUTO_SAVE=1)…');
+    const saveResult = await client.build.workflows.save_workflow_bundle(projectId, {
+      files: bundle.files,
+      dry_run: false,
+    });
+    console.error('Deploying…');
+    const deployResult = await client.build.workflows.deploy({
+      project_id: projectId,
+      instance_id: instanceId,
+    });
+    console.log(JSON.stringify({ save: saveResult, deploy: deployResult }, null, 2));
+    console.error(`
+Provisioned. Write events:
+  LOXTEP_DATA_PRODUCT_NAME=${DATA_PRODUCT_NAME} node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+`);
+    return;
+  }
+
+  console.error(`
+Next steps (no MCP required)
+────────────────────────────
+Preferred — one command:
+  pnpm exec loxtep ingest provision --name ${DATA_PRODUCT_NAME}
+
+Or save the generated bundle:
+  pnpm exec loxtep bundle save --dry-run --file ${outPath}
+  pnpm exec loxtep bundle save --file ${outPath}
+  pnpm exec loxtep workflows deploy --project-id ${projectId} --instance-id ${instanceId}
+
+Write events:
+  LOXTEP_DATA_PRODUCT_NAME=${DATA_PRODUCT_NAME} node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+`);
+}
+
+main().catch(err => {
+  console.error(err instanceof Error ? err.message : err);
+  process.exit(1);
+});

package/docs/examples/write-events.mjs

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+/**
+ * Write sample events to a deployed source data product via get_writer.
+ *
+ * Prerequisites: login, init, attach, SDK ingest workflow deployed (see sdk-first-ingest.md).
+ *
+ *   node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+ *
+ * Env:
+ *   LOXTEP_DATA_PRODUCT_NAME  default: app-events
+ */
+
+import { LoxtepClient } from '@loxtep/sdk';
+
+const DATA_PRODUCT_NAME = process.env.LOXTEP_DATA_PRODUCT_NAME ?? 'app-events';
+
+async function main() {
+  const client = await LoxtepClient.fromWorkspace();
+
+  console.error(`Opening writer for data product "${DATA_PRODUCT_NAME}"…`);
+  const writer = await client.get_writer(DATA_PRODUCT_NAME);
+
+  const events = [
+    {
+      event_id: `evt_${Date.now()}_1`,
+      occurred_at: new Date().toISOString(),
+      payload: { source: 'write-events.mjs', action: 'example_write', index: 1 },
+    },
+    {
+      event_id: `evt_${Date.now()}_2`,
+      occurred_at: new Date().toISOString(),
+      payload: { source: 'write-events.mjs', action: 'example_write', index: 2 },
+    },
+  ];
+
+  for (const event of events) {
+    writer.write(event);
+    console.log(JSON.stringify({ written: event.event_id }));
+  }
+
+  await writer.close();
+  console.error('Writer closed — events flushed.');
+}
+
+main().catch(err => {
+  const message = err instanceof Error ? err.message : String(err);
+  console.error(message);
+  if (/not deployed/i.test(message)) {
+    console.error(
+      '\nDeploy an SDK ingestion workflow first. See nodejs/docs/sdk-first-ingest.md'
+    );
+  }
+  process.exit(1);
+});

package/docs/getting-started.md

@@ -1,9 +1,13 @@
 # SDK Getting Started Guide
 
 Get from zero to a working Loxtep workspace: authenticate, scaffold a project,
-attach to a runtime instance, and generate typed constants. Stream read/write
-comes **after** you deploy a workflow that publishes a data product.
+attach to a runtime instance, **provision your first source data product**, and
+write events with `get_writer`.
 
+> **SDK-first ingest (login → init → attach → write):**
+> [SDK-first ingest](./sdk-first-ingest.md) — the primary path when your app
+> sends events via the SDK (no SaaS connector first).
+>
 > **Other paths:** [Code-first CLI](./code-first-cli.md) (`init → attach →
 > generate → test → deploy`), [Agent-first MCP](https://github.com/LoxtepInc/loxtep-plugins-skills),
 > or the **Web UI** at [app.loxtep.io](https://app.loxtep.io).
@@ -17,9 +21,10 @@ comes **after** you deploy a workflow that publishes a data product.
 - [Step 3: Scaffold a Workspace](#step-3-scaffold-a-workspace)
 - [Step 4: Attach to an Instance](#step-4-attach-to-an-instance)
 - [Step 5: Generate Typed Constants](#step-5-generate-typed-constants)
-- [Step 6: Explore the Platform](#step-6-explore-the-platform)
-- [Step 7: Use the SDK in Application Code](#step-7-use-the-sdk-in-application-code)
-- [Step 8: Write and Read Events (after deploy)](#step-8-write-and-read-events-after-deploy)
+- [Step 6: Your first data product (SDK ingest)](#step-6-your-first-data-product-sdk-ingest)
+- [Step 7: Explore the platform (optional)](#step-7-explore-the-platform-optional)
+- [Step 8: Use the SDK in application code](#step-8-use-the-sdk-in-application-code)
+- [Step 9: Write and read events](#step-9-write-and-read-events)
 - [Troubleshooting](#troubleshooting)
 - [Next Steps](#next-steps)
 
@@ -158,7 +163,33 @@ products, connectors, domains, queues, workflows). Re-run after platform changes
 
 ---
 
-## Step 6: Explore the Platform
+## Step 6: Your first data product (SDK ingest)
+
+After `attach`, the usual greenfield goal is: **provision a source data product
+on your instance, then call `get_writer` from your app.**
+
+`get_writer` needs **deployment metadata** (queues and bots created at deploy
+time). Creating a catalog row alone is not enough.
+
+**Recommended:** follow **[SDK-first ingest](./sdk-first-ingest.md)** end to end.
+Short version:
+
+```bash
+pnpm exec loxtep domains list
+
+# One command: SDK connector + workflow bundle + deploy (no MCP)
+pnpm exec loxtep ingest provision --name app-events
+
+pnpm exec loxtep data-products list
+node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+```
+
+Do **not** expect `pnpm exec loxtep data-products create` alone to enable
+streaming — it registers catalog metadata without runtime bindings.
+
+---
+
+## Step 7: Explore the platform (optional)
 
 Run these **from your workspace directory** (where `.loxtep/project.json` lives)
 after Steps 3–4:
@@ -178,12 +209,12 @@ pnpm exec loxtep projects list
 pnpm exec loxtep workflows list --project-id <project-id-from-list>
 ```
 
-On a new account, **data products may be empty** until you create and deploy
-workflows (CLI, MCP, or Web UI). That's expected — don't call `get_writer` yet.
+On a new account, **data products may be empty** until you complete
+[Step 6](#step-6-your-first-data-product-sdk-ingest). That's expected.
 
 ---
 
-## Step 7: Use the SDK in Application Code
+## Step 8: Use the SDK in application code
 
 Prefer workspace auto-config instead of hand-building `api_url` and tokens:
 
@@ -209,33 +240,22 @@ const instances = await client.workspace.instances.list();
 
 ---
 
-## Step 8: Write and Read Events (after deploy)
-
-`get_writer` and `get_reader` resolve queue, bot, and stream configuration from
-**deployment metadata**. They only work for data products that exist **and** are
-deployed on your attached instance.
-
-1. Author a workflow under `workflows/` (see [Code-first CLI](./code-first-cli.md)).
-2. `pnpm exec loxtep deploy`
-3. Confirm the data product name:
-
-   ```bash
-   pnpm exec loxtep data-products list
-   ```
+## Step 9: Write and read events
 
-4. Stream from application code:
+After [Step 6](#step-6-your-first-data-product-sdk-ingest), stream from
+application code:
 
 ```typescript
 const client = await LoxtepClient.fromWorkspace();
 
-const writer = await client.get_writer('orders'); // use your deployed name
+const writer = await client.get_writer('app-events'); // name from Step 6 / data-products list
 writer.write({
   order_id: 'ord_1',
   total: 99.5,
 });
 await writer.close();
 
-const reader = await client.get_reader('orders', {
+const reader = await client.get_reader('app-events', {
   bot_id: 'my-app-reader',
 });
 for await (const event of reader) {
@@ -318,6 +338,7 @@ data product UUID instead of a name.
 
 | Resource | Description |
 |----------|-------------|
+| [SDK-first ingest](./sdk-first-ingest.md) | Greenfield: provision + `get_writer` |
 | [Code-first CLI guide](./code-first-cli.md) | Workflow modules, `test`, `deploy` |
 | [Quick Reference Card](./quick-reference.md) | Cheat sheet for common operations |
 | [Event Replay Cookbook](./event-replay-cookbook.md) | Replay and reprocess historical events |

package/docs/quick-reference.md

@@ -1,8 +1,8 @@
 # SDK Quick Reference Card
 
 Concise cheat sheet for common Loxtep SDK operations (Node.js).
-For full walkthroughs, see the [Getting Started Guide](./getting-started.md)
-or [Code-first CLI guide](./code-first-cli.md).
+For full walkthroughs, see [SDK-first ingest](./sdk-first-ingest.md),
+[Getting Started Guide](./getting-started.md), or [Code-first CLI guide](./code-first-cli.md).
 
 ---
 
@@ -183,10 +183,24 @@ const hits = await client.query.catalog.search({ query: 'orders' });
 Workspace lifecycle (requires `loxtep init` first — see [Code-first CLI guide](./code-first-cli.md)):
 
 ```bash
-pnpm exec loxtep init [--template shopify-orders]
 pnpm exec loxtep login
-pnpm exec loxtep attach --instance prod
+pnpm exec loxtep init
+pnpm exec loxtep attach --instance <instance-id>
 pnpm exec loxtep generate
+```
+
+**SDK-first ingest** (after attach — see [SDK-first ingest](./sdk-first-ingest.md)):
+
+```bash
+pnpm exec loxtep domains list
+node node_modules/@loxtep/sdk/docs/examples/generate-ingest-bundle.mjs
+pnpm exec loxtep workflows deploy --project-id <id> --instance-id <id>
+node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+```
+
+Code-first workflow modules:
+
+```bash
 pnpm exec loxtep test my-module --event ./events/sample.json
 pnpm exec loxtep deploy
 ```

package/docs/sdk-first-ingest.md

@@ -0,0 +1,304 @@
+# SDK-first ingest — your first data product and `get_writer`
+
+You logged in, ran `init`, and `attach`. This guide takes you from there to
+**writing events from application code** with `client.get_writer(name)`.
+
+That is the primary greenfield path when you are **not** wiring a SaaS connector
+first — you want your app (or ETL job) to push events into a **source** data
+product on your attached instance.
+
+> **Full CLI workspace setup:** [Getting Started](./getting-started.md) steps 1–4.
+> **Code-first workflow modules:** [Code-first CLI](./code-first-cli.md).
+
+---
+
+## What you are building
+
+```text
+Your app  ──get_writer──▶  rstreams queue  ──▶  source data product (catalog)
+     ▲                           ▲
+     │                           │
+  @loxtep/sdk              provisioned at deploy time
+  fromWorkspace()
+```
+
+`get_writer` resolves **queue**, **bot**, and **stream bus** config from
+**deployment metadata** (`deployment_bindings` on the data product). A catalog
+row alone is not enough — the ingestion workflow must be **saved and deployed**
+on your attached instance first.
+
+---
+
+## Prerequisites
+
+Run from your scaffolded workspace (directory with `.loxtep/project.json`):
+
+```bash
+pnpm exec loxtep config list   # project_id + instance_id should be set
+pnpm exec loxtep domains list  # you need a domain_id for the data product
+```
+
+Every org has at least one domain. Copy a `domain_id` from the list.
+
+---
+
+## Overview (four phases)
+
+| Phase | What | How |
+| ----- | ---- | --- |
+| 1 | Workspace ready | `login` → `init` → `attach` (done) |
+| 2 | Provision runtime | SDK connector + workflow bundle + deploy |
+| 3 | Verify | `data-products list` / `data-products get` |
+| 4 | Write events | `LoxtepClient.fromWorkspace()` + `get_writer` |
+
+Phases 2–4 below.
+
+---
+
+## Phase 2 — Provision runtime (required before `get_writer`)
+
+Runtime provisioning creates the workflow graph (SDK connection node → source
+data product node), saves it to your project workspace, and **deploys** it to your
+instance so queues and bots exist.
+
+### Option A — CLI one-shot (recommended, no MCP)
+
+From your workspace root (after `login`, `init`, `attach`):
+
+```bash
+pnpm exec loxtep ingest provision --name app-events
+```
+
+This command:
+
+1. Creates an org-level **SDK connector** (`connector_type: "sdk"`).
+2. Builds and saves the workflow bundle (`workflow.json` + connection + source data product).
+3. **Deploys** to your attached instance (unless you pass `--no-deploy` or `--dry-run`).
+
+Validate first without writing:
+
+```bash
+pnpm exec loxtep ingest provision --name app-events --dry-run
+```
+
+### Option B — Helper script + bundle save
+
+Generate bundle JSON only:
+
+```bash
+node node_modules/@loxtep/sdk/docs/examples/generate-ingest-bundle.mjs
+```
+
+Then persist and deploy via CLI:
+
+```bash
+pnpm exec loxtep bundle save --dry-run --file .loxtep/sdk-ingest-bundle.json
+pnpm exec loxtep bundle save --file .loxtep/sdk-ingest-bundle.json
+pnpm exec loxtep workflows deploy \
+  --project-id <project-id-from-config-list> \
+  --instance-id <instance-id-from-config-list>
+```
+
+Or set `LOXTEP_AUTO_SAVE=1` on the helper script to save + deploy in one step.
+
+Poll until deployment completes (Web UI **Observe**, or `loxtep observe status`).
+
+### Option C — Loxtep MCP (Cursor / agent)
+
+Use **`loxtep_build`** → `save_workflow_bundle` if you prefer MCP authoring.
+Same bundle shape as below; deploy with `loxtep workflows deploy` or MCP
+`deploy_project`.
+
+### Option D — Web UI (Studio)
+
+Create an **ingestion** workflow with an **SDK** connection node and a **source**
+data product node, then deploy to your attached instance from the Studio deploy
+flow.
+
+---
+
+### Workflow bundle shape
+
+Minimal ingestion bundle (SDK connection → source data product). Replace UUIDs
+and names; `connector_id` must be the SDK connector from step 2A or 2B.
+
+```json
+{
+  "workflow.json": {
+    "workflow_id": "<workflow-uuid>",
+    "organization_id": "<org-uuid>",
+    "project_id": "<project-uuid>",
+    "name": "SDK App Events Ingest",
+    "workflow_type": "ingestion",
+    "domain_id": "<domain-uuid>",
+    "status": "active",
+    "configuration": {},
+    "metadata": {},
+    "created_at": "2026-07-23T00:00:00.000Z",
+    "updated_at": "2026-07-23T00:00:00.000Z"
+  },
+  "connections/<connection-uuid>.json": {
+    "connection_id": "<connection-uuid>",
+    "organization_id": "<org-uuid>",
+    "project_id": "<project-uuid>",
+    "workflow_id": "<workflow-uuid>",
+    "connector_id": "<sdk-connector-uuid>",
+    "key": "sdk-input",
+    "name": "SDK Input",
+    "type": "sdk",
+    "status": "active",
+    "configuration": {
+      "sdk_type": "nodejs",
+      "event_type": "app-events"
+    },
+    "created_at": "2026-07-23T00:00:00.000Z",
+    "updated_at": "2026-07-23T00:00:00.000Z"
+  },
+  "data-products/<data-product-uuid>.json": {
+    "data_product_id": "<data-product-uuid>",
+    "organization_id": "<org-uuid>",
+    "project_id": "<project-uuid>",
+    "workflow_id": "<workflow-uuid>",
+    "upstream_entity_id": "<connection-uuid>",
+    "upstream_entity_type": "connections",
+    "domain_id": "<domain-uuid>",
+    "name": "app-events",
+    "kind": "source",
+    "status": "draft",
+    "owner": {},
+    "governance": {
+      "classification": "internal",
+      "pii_fields": [],
+      "compliance_requirements": [],
+      "tags": []
+    },
+    "metadata": {},
+    "created_at": "2026-07-23T00:00:00.000Z",
+    "updated_at": "2026-07-23T00:00:00.000Z"
+  }
+}
+```
+
+The **`name`** on the data product node (`app-events` above) is what you pass
+to `get_writer('app-events')`.
+
+---
+
+### Do not stop at `data-products create`
+
+```bash
+# Creates a catalog row only — get_writer will fail with "not deployed"
+pnpm exec loxtep data-products create --name app-events --domain-id <uuid> --kind source
+```
+
+Use **`data-products create`** only when you already have (or will immediately
+add) a deployed workflow that sets `deployment_bindings`. For greenfield SDK
+ingest, prefer the bundle + deploy path above.
+
+---
+
+## Phase 3 — Verify deployment
+
+```bash
+pnpm exec loxtep data-products list
+pnpm exec loxtep data-products get <data-product-id>
+```
+
+A deployable source data product includes **`deployment_bindings`** with
+`queue_name`, `bot_id`, and `instance_id`. If those are missing, finish Phase 2
+before writing.
+
+Re-run code generation after platform changes:
+
+```bash
+pnpm exec loxtep generate
+```
+
+---
+
+## Phase 4 — Write events from your app
+
+Copy or run the example:
+
+```bash
+node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+```
+
+Or embed in your service:
+
+```typescript
+import { LoxtepClient } from '@loxtep/sdk';
+
+const client = await LoxtepClient.fromWorkspace();
+
+// Name from Phase 2 bundle / data-products list (case-sensitive)
+const writer = await client.get_writer('app-events');
+
+writer.write({
+  event_id: 'evt_001',
+  occurred_at: new Date().toISOString(),
+  payload: { user_id: 'u_1', action: 'signup' },
+});
+
+await writer.close(); // flush buffered events
+```
+
+Optional writer tuning:
+
+```typescript
+await client.get_writer('app-events', {
+  batch_size: 500,
+  max_retries: 5,
+});
+```
+
+Read back (after data has landed):
+
+```typescript
+const reader = await client.get_reader('app-events', {
+  bot_id: 'my-app-reader',
+});
+for await (const event of reader) {
+  console.log(event);
+  break;
+}
+```
+
+---
+
+## End-to-end checklist
+
+```bash
+pnpm exec loxtep login
+pnpm exec loxtep init
+pnpm exec loxtep attach --instance <instance-id>
+pnpm exec loxtep domains list
+
+pnpm exec loxtep ingest provision --name app-events
+# or: generate-ingest-bundle.mjs + loxtep bundle save
+
+pnpm exec loxtep data-products list
+node node_modules/@loxtep/sdk/docs/examples/write-events.mjs
+```
+
+---
+
+## Troubleshooting
+
+| Symptom | Cause | Fix |
+| ------- | ----- | --- |
+| `StreamingError: … is not deployed` | No `deployment_bindings` | Complete Phase 2 (bundle + deploy) |
+| `NotFoundError: Data product '…' not found` | Wrong name or wrong instance | `data-products list`; check `attach` / `instance_id` |
+| `AmbiguityError: Multiple data products match` | Same name on multiple instances | Use UUID or set `instance_id` in project.json |
+| Empty `domains list` | New org / permissions | Create a domain in the Web UI (Governance) |
+
+---
+
+## Next steps
+
+| Resource | Description |
+| -------- | ----------- |
+| [Getting Started](./getting-started.md) | Login, init, attach, generate |
+| [Quick Reference](./quick-reference.md) | CLI + SDK cheat sheet |
+| [Code-first CLI](./code-first-cli.md) | TypeScript workflow modules + `loxtep deploy` |
+| [Event Replay Cookbook](./event-replay-cookbook.md) | Replay and reprocess |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loxtep/sdk",
-  "version": "0.7.15",
+  "version": "0.7.18",
   "description": "Loxtep SDK for Node.js — the Enterprise Context Layer: data products, workflows, projects, queues, and machine-usable context for AI",
   "author": "Loxtep <engineering@loxtep.io>",
   "homepage": "https://loxtep.io",