@nanobpm/urban 0.51.0 → 0.52.0

package/dist/runtime/core/modules/workers.js

@@ -5,6 +5,8 @@ import { isRecord } from "../guards.js";
 import { workerJobType } from "../manifest.js";
 import { runLlmJob } from "./llm.js";
 import { runInJobContext } from "../execContext.js";
+import { readLineage } from "../lineage.js";
+import { LineageStore } from "./lineage-store.js";
 /** Adapt an engine SDK client into the LLM module's `DecisionEvaluator`, parsing the
  *  decision's JSON-string `output` into the value fed back to the LLM job. Validates the
  *  SDK shape and adds decision context to a parse failure rather than surfacing a raw
@@ -48,6 +50,59 @@ function withJobLog(app, job) {
         bindings.elementId = job.elementId;
     return { ...app, log: app.log.child(bindings) };
 }
+/**
+ * Build the ambient {@link JobExecContext} a worker runs its handler within. Beyond the
+ * instance/element/jobType used for write-provenance, it resolves the lineage
+ * `rootRequestKey` (issue #254): the job's own `_urban.lineage` envelope root when present,
+ * else the instance's own key — so an instance with no envelope is treated as a root and any
+ * instance/message its handler spawns inherits that root with `causedByInstanceKey` set.
+ */
+function jobExecContext(job, jobType) {
+    return {
+        instanceKey: job.processInstanceKey,
+        elementId: job.elementId,
+        jobType,
+        rootRequestKey: readLineage(job.variables)?.rootRequestKey ?? job.processInstanceKey,
+    };
+}
+/**
+ * Build the framework lineage projection store over the app's DEFAULT data source, or `undefined`
+ * when the app configures none. The store records the `_urban.lineage` edge every activated job
+ * reveals (issue #254), so intent→progress lineage materialises for free — but it is a sidecar on
+ * the app's own source (like write-provenance), so an app with no datasource simply records
+ * nothing. Absent- and error-safe: any failure to provision degrades to no lineage recording, never
+ * a failed worker mount.
+ */
+function tryLineageStore(app, log) {
+    if (!app.data.hasDefaultSource()) {
+        // No default data source configured at all — lineage is a sidecar on the app's own source, so
+        // this is the expected absent case (like write-provenance): record nothing, silently.
+        log("debug", "lineage: no default data source; lineage recording disabled");
+        return undefined;
+    }
+    try {
+        const source = app.data.source();
+        const store = new LineageStore(source.db);
+        store.ensureSchema();
+        return store;
+    }
+    catch (err) {
+        // A default IS configured but resolving or provisioning it failed — a missing named source
+        // (`no such data source`) or a schema/DB error. That is a genuine fault, not the absent case,
+        // so surface it at `warn` rather than disguising it as "no source".
+        log("warn", "lineage: failed to provision projection store; lineage recording disabled", { error: String(err) });
+        return undefined;
+    }
+}
+/** Record an activated job's lineage edge, never letting a projection write break the job. */
+function recordJobLineage(store, job, log) {
+    try {
+        store.recordFromJob(job);
+    }
+    catch (err) {
+        log("warn", "lineage: failed to record job edge", { jobType: job.jobType, error: String(err) });
+    }
+}
 /**
  * Resolve a handler for `jobType` from a loaded module, in priority order:
  *   1. `handlers[jobType]`            (a map keyed by job type — the multi-type module case)
@@ -74,6 +129,10 @@ export async function mountWorkers(ctx, app) {
     const decls = ctx.manifest.workers ?? [];
     const subs = [];
     const jobTypes = [];
+    // The framework lineage projection (issue #254): each activated job records the `_urban.lineage`
+    // edge its instance carries, so intent→progress lineage materialises for free. Absent when the
+    // app configures no default data source.
+    const lineageStore = tryLineageStore(app, ctx.host.log);
     // Cache module loads so a multi-type handler module is imported once.
     const moduleCache = new Map();
     const joinRoot = (p) => p.startsWith("/") ? p : `${ctx.root.replace(/\/+$/, "")}/${p.replace(/^\/+/, "")}`;
@@ -114,7 +173,11 @@ export async function mountWorkers(ctx, app) {
                 env: (n) => app.env(n),
                 evaluateDecision: app.sdk ? sdkDecisionEvaluator(app.sdk) : undefined,
             };
-            const wrapped = (job) => runInJobContext({ instanceKey: job.processInstanceKey, elementId: job.elementId, jobType }, () => runLlmJob(job.variables, binding, rt));
+            const wrapped = (job) => {
+                if (lineageStore)
+                    recordJobLineage(lineageStore, job, ctx.host.log);
+                return runInJobContext(jobExecContext(job, jobType), () => runLlmJob(job.variables, binding, rt));
+            };
             const sub = await ctx.engine.registerWorker(jobType, wrapped, {
                 workerName: `${ctx.manifest.id}:${jobType}`,
             });
@@ -129,7 +192,11 @@ export async function mountWorkers(ctx, app) {
             throw new Error(`worker "${jobType}": ${decl.handler} exports no handler for it ` +
                 `(expected handlers["${jobType}"], a named export, or a default function)`);
         }
-        const wrapped = (job) => runInJobContext({ instanceKey: job.processInstanceKey, elementId: job.elementId, jobType }, () => handler(job, withJobLog(app, job)));
+        const wrapped = (job) => {
+            if (lineageStore)
+                recordJobLineage(lineageStore, job, ctx.host.log);
+            return runInJobContext(jobExecContext(job, jobType), () => handler(job, withJobLog(app, job)));
+        };
         const sub = await ctx.engine.registerWorker(jobType, wrapped, {
             workerName: `${ctx.manifest.id}:${jobType}`,
         });

package/dist/runtime/engine/nanosdk.js

@@ -21,6 +21,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { isBpmnError } from "../core/host.js";
 import { buildFormSchema, parseFormSchema, pickFormLinkage, resolveFormIdentifier, } from "../core/form-contract.js";
 import { isRecord } from "../core/guards.js";
+import { applyAmbientLineage } from "../core/lineage.js";
 /** Coerce an engine response's process-instance key to a non-empty string, or
  * throw — a missing key means a malformed/partial response, not a real instance. */
 export function requireProcessInstanceKey(key) {
@@ -76,9 +77,12 @@ export class SdkEngineClient {
         return { deployed: resources.length };
     }
     async createInstance(input) {
+        // Auto-thread the `_urban.lineage` envelope (issue #254): a handler that spawns an
+        // instance propagates its `rootRequestKey` and stamps `causedByInstanceKey`; a genuine
+        // top-level request mints a fresh root. An explicit caller-supplied envelope wins.
         const body = await this.client.createProcessInstance({
             processDefinitionId: input.processDefinitionId,
-            variables: input.variables ?? {},
+            variables: applyAmbientLineage(input.variables),
             awaitCompletion: input.awaitCompletion ?? false,
         });
         const rawKey = body.processInstanceKey ?? body.key;
@@ -93,10 +97,12 @@ export class SdkEngineClient {
         await this.client.cancelProcessInstance({ processInstanceKey: input.processInstanceKey });
     }
     async publishMessage(input) {
+        // Message-started instances inherit lineage too (issue #254): thread the envelope onto the
+        // message variables so the instance the message starts carries `_urban.lineage`.
         await this.client.publishMessage({
             name: input.name,
             correlationKey: input.correlationKey ?? "",
-            variables: input.variables ?? {},
+            variables: applyAmbientLineage(input.variables),
         });
     }
     async searchUserTasks(filter) {

package/dist/runtime/index.d.ts

@@ -31,6 +31,10 @@ export { resolveHandler, sdkDecisionEvaluator } from "./core/modules/workers.ts"
 export { buildMessages, callLlm, resolveProvider, runLlmJob, } from "./core/modules/llm.ts";
 export type { ChatMessage, DecisionEvaluator, EnvLookup, LlmRuntime, LlmVars, ProviderConfig, } from "./core/modules/llm.ts";
 export { evalCorrelation } from "./core/modules/triggers.ts";
+export { applyAmbientLineage, buildLineageTree, deriveLineage, LINEAGE_KEY, LINEAGE_NAMESPACE, mintRootRequestKey, readLineage, writeLineage, } from "./core/lineage.ts";
+export type { ApplyLineageOptions, LineageAttachment, LineageEdge, LineageEdgeType, LineageEnvelope, LineageNode, LineageTree, } from "./core/lineage.ts";
+export { LINEAGE_ATTACHMENTS_TABLE, LINEAGE_EDGES_TABLE, LINEAGE_SCHEMA_SQL, LineageStore, systemClock as lineageSystemClock, } from "./core/modules/lineage-store.ts";
+export type { Clock as LineageClock, LineageJobLike, LineageStoreOptions, } from "./core/modules/lineage-store.ts";
 export { deployModels, RESOURCES_DIR } from "./core/modules/deploy.ts";
 export { defaultScheduler, MAX_TIMER_DELAY_MS } from "./core/modules/scheduler.ts";
 export type { SchedulerDeps } from "./core/modules/scheduler.ts";

package/dist/runtime/index.js

@@ -28,6 +28,11 @@ export { defineOperation, mountApi, resolveOperationHandler, NotImplemented } fr
 export { resolveHandler, sdkDecisionEvaluator } from "./core/modules/workers.js";
 export { buildMessages, callLlm, resolveProvider, runLlmJob, } from "./core/modules/llm.js";
 export { evalCorrelation } from "./core/modules/triggers.js";
+// Lineage — the framework-level lineage primitive (issue #254): the `_urban.lineage`
+// envelope + auto-threading, and the generic read projection that stitches user intent →
+// progress (a root request → the instances/PRs/tasks it causes).
+export { applyAmbientLineage, buildLineageTree, deriveLineage, LINEAGE_KEY, LINEAGE_NAMESPACE, mintRootRequestKey, readLineage, writeLineage, } from "./core/lineage.js";
+export { LINEAGE_ATTACHMENTS_TABLE, LINEAGE_EDGES_TABLE, LINEAGE_SCHEMA_SQL, LineageStore, systemClock as lineageSystemClock, } from "./core/modules/lineage-store.js";
 // Model deploy (deploy-by-convention under `resources/`, ADR 0062)
 export { deployModels, RESOURCES_DIR } from "./core/modules/deploy.js";
 export { defaultScheduler, MAX_TIMER_DELAY_MS } from "./core/modules/scheduler.js";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nanobpm/urban",
-	"version": "0.51.0",
+	"version": "0.52.0",
 	"description": "Urban: build and run code-first apps on Nano — runtime, derivation toolkit, and CLI in one. Author durable processes with @nanobpm/workflow (defineFlow), a typed datasource, forms, triggers and surfaces from a nano.app.json manifest, on Node or Deno.",
 	"license": "Apache-2.0",
 	"type": "module",

package/src/runtime/core/execContext.ts

@@ -21,6 +21,11 @@ export interface JobExecContext {
   elementId?: string;
   /** The job type (worker) currently executing. */
   jobType?: string;
+  /** The lineage `rootRequestKey` the running instance belongs to (its own envelope's root,
+   *  or — for a root instance with no envelope — its own instance key). Threaded so that when a
+   *  handler spawns an instance or publishes a message, the SDK propagates the same root and
+   *  sets `causedByInstanceKey` to {@link instanceKey} (issue #254, see `./lineage.ts`). */
+  rootRequestKey?: string;
 }
 
 let store: AsyncStore<JobExecContext> | undefined;

package/src/runtime/core/lineage.test.ts

@@ -0,0 +1,215 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+  applyAmbientLineage,
+  buildLineageTree,
+  deriveLineage,
+  type LineageEdge,
+  LINEAGE_KEY,
+  LINEAGE_NAMESPACE,
+  mintRootRequestKey,
+  readLineage,
+  writeLineage,
+} from "./lineage.ts";
+import { __resetExecStoreForTests, installExecStore, runInJobContext } from "./execContext.ts";
+import { createNodeHost } from "../adapters/node.ts";
+import { isRecord } from "./guards.ts";
+
+function lineageOf(variables: Record<string, unknown>): unknown {
+  const ns = variables[LINEAGE_NAMESPACE];
+  return isRecord(ns) ? ns[LINEAGE_KEY] : undefined;
+}
+
+test("readLineage extracts a well-formed envelope and rejects malformed ones", () => {
+  assert.deepEqual(
+    readLineage({ _urban: { lineage: { rootRequestKey: "r1", causedByInstanceKey: "c1" } } }),
+    { rootRequestKey: "r1", causedByInstanceKey: "c1" },
+  );
+  // A bare root (no cause) is a valid fresh root.
+  assert.deepEqual(readLineage({ _urban: { lineage: { rootRequestKey: "r1" } } }), {
+    rootRequestKey: "r1",
+    causedByInstanceKey: undefined,
+  });
+  // Blank/absent rootRequestKey → not a valid envelope.
+  assert.equal(readLineage({ _urban: { lineage: { rootRequestKey: "" } } }), undefined);
+  assert.equal(readLineage({ _urban: { lineage: {} } }), undefined);
+  assert.equal(readLineage({ _urban: {} }), undefined);
+  assert.equal(readLineage({}), undefined);
+  assert.equal(readLineage(undefined), undefined);
+});
+
+test("writeLineage never mutates input and preserves other _urban.* keys", () => {
+  const input = { keep: 1, _urban: { other: "x" } };
+  const out = writeLineage(input, { rootRequestKey: "r1", causedByInstanceKey: "c1" });
+  assert.deepEqual(input, { keep: 1, _urban: { other: "x" } }, "input is not mutated");
+  assert.equal(out.keep, 1);
+  assert.deepEqual(out._urban, { other: "x", lineage: { rootRequestKey: "r1", causedByInstanceKey: "c1" } });
+});
+
+test("writeLineage drops an absent causedByInstanceKey (a fresh root is { rootRequestKey })", () => {
+  const out = writeLineage({}, { rootRequestKey: "r1" });
+  assert.deepEqual(lineageOf(out), { rootRequestKey: "r1" });
+});
+
+test("deriveLineage propagates an ambient root and sets causedByInstanceKey", () => {
+  assert.deepEqual(deriveLineage({ rootRequestKey: "root-1", instanceKey: "inst-5" }), {
+    rootRequestKey: "root-1",
+    causedByInstanceKey: "inst-5",
+  });
+});
+
+test("deriveLineage treats an ambient instance with no envelope as its own root", () => {
+  assert.deepEqual(deriveLineage({ instanceKey: "inst-9" }), {
+    rootRequestKey: "inst-9",
+    causedByInstanceKey: "inst-9",
+  });
+});
+
+test("deriveLineage mints a fresh root when there is no ambient job at all", () => {
+  assert.deepEqual(deriveLineage(undefined, () => "minted-1"), { rootRequestKey: "minted-1" });
+});
+
+test("mintRootRequestKey returns a non-empty, unique-ish value", () => {
+  const a = mintRootRequestKey();
+  const b = mintRootRequestKey();
+  assert.equal(typeof a, "string");
+  assert.notEqual(a, "");
+  assert.notEqual(a, b);
+});
+
+test("applyAmbientLineage: an explicit caller envelope always wins (untouched)", () => {
+  const explicit = { _urban: { lineage: { rootRequestKey: "explicit", causedByInstanceKey: "x" } } };
+  const out = applyAmbientLineage(explicit, {
+    ambient: { rootRequestKey: "ambient-root", instanceKey: "ambient-inst" },
+  });
+  assert.deepEqual(lineageOf(out), { rootRequestKey: "explicit", causedByInstanceKey: "x" });
+});
+
+test("applyAmbientLineage: propagates the ambient root and stamps the cause", () => {
+  const out = applyAmbientLineage(
+    { payload: 1 },
+    { ambient: { rootRequestKey: "root-A", instanceKey: "inst-B" } },
+  );
+  assert.equal(out.payload, 1);
+  assert.deepEqual(lineageOf(out), { rootRequestKey: "root-A", causedByInstanceKey: "inst-B" });
+});
+
+test("applyAmbientLineage: mints a fresh root for a genuine top-level request", () => {
+  const out = applyAmbientLineage({}, { ambient: undefined, mintRootRequestKey: () => "fresh-1" });
+  assert.deepEqual(lineageOf(out), { rootRequestKey: "fresh-1" });
+});
+
+test("applyAmbientLineage reads the real ambient job context (runInJobContext)", () => {
+  __resetExecStoreForTests();
+  const host = createNodeHost({ cwd: process.cwd(), log: () => {} });
+  installExecStore(() => host.createAsyncStore?.());
+  try {
+    const out = runInJobContext({ instanceKey: "pi-1", rootRequestKey: "root-1", jobType: "w" }, () =>
+      applyAmbientLineage({ a: 1 }),
+    );
+    assert.deepEqual(lineageOf(out), { rootRequestKey: "root-1", causedByInstanceKey: "pi-1" });
+  } finally {
+    __resetExecStoreForTests();
+  }
+});
+
+test("buildLineageTree stitches a fan-out tree from weak edges", () => {
+  const edges: LineageEdge[] = [
+    { rootRequestKey: "root", instanceKey: "root", edgeType: "weak" },
+    { rootRequestKey: "root", instanceKey: "child-1", causedByInstanceKey: "root", edgeType: "weak" },
+    { rootRequestKey: "root", instanceKey: "child-2", causedByInstanceKey: "root", edgeType: "weak" },
+    { rootRequestKey: "root", instanceKey: "grand-1", causedByInstanceKey: "child-1", edgeType: "weak" },
+  ];
+  const tree = buildLineageTree("root", edges);
+  assert.equal(tree.root.instanceKey, "root");
+  assert.deepEqual(tree.root.children.map((c) => c.instanceKey).sort(), ["child-1", "child-2"]);
+  const child1 = tree.root.children.find((c) => c.instanceKey === "child-1");
+  assert.deepEqual(child1?.children.map((c) => c.instanceKey), ["grand-1"]);
+  assert.equal(tree.nodes.length, 4);
+});
+
+test("buildLineageTree: a strong (engine) edge supersedes a weak edge on the same node", () => {
+  const edges: LineageEdge[] = [
+    { rootRequestKey: "root", instanceKey: "child", causedByInstanceKey: "root", edgeType: "weak" },
+    { rootRequestKey: "root", instanceKey: "child", causedByInstanceKey: "root", edgeType: "strong" },
+  ];
+  const tree = buildLineageTree("root", edges);
+  const child = tree.nodes.find((n) => n.instanceKey === "child");
+  assert.equal(child?.edgeType, "strong");
+});
+
+test("buildLineageTree ignores edges from other roots and hangs attachments on their node", () => {
+  const edges: LineageEdge[] = [
+    { rootRequestKey: "root", instanceKey: "child", causedByInstanceKey: "root", edgeType: "weak" },
+    { rootRequestKey: "other", instanceKey: "stranger", causedByInstanceKey: "other", edgeType: "weak" },
+  ];
+  const tree = buildLineageTree("root", edges, [
+    { nodeKey: "child", kind: "pull_request", ref: "owner/repo#1", label: "PR 1" },
+    { nodeKey: "missing", kind: "task", ref: "t-1" },
+  ]);
+  assert.equal(tree.nodes.some((n) => n.instanceKey === "stranger"), false);
+  const child = tree.root.children[0];
+  assert.equal(child.instanceKey, "child");
+  assert.deepEqual(child.attachments, [{ nodeKey: "child", kind: "pull_request", ref: "owner/repo#1", label: "PR 1" }]);
+  // An attachment for an instance with no recorded edge yet is PRESERVED as an unattached node,
+  // never silently dropped — attachments are an extension point and must survive the projection.
+  const orphan = tree.nodes.find((n) => n.instanceKey === "missing");
+  assert.ok(orphan, "attachment for an un-stitched instance must materialise its node");
+  assert.deepEqual(orphan.attachments, [{ nodeKey: "missing", kind: "task", ref: "t-1" }]);
+  assert.equal(orphan.edgeType, undefined);
+  assert.equal(orphan.children.length, 0);
+});
+
+test("buildLineageTree: a stray orphan attachment does not shadow a MINTED root's real parentless instance", () => {
+  // Defect-class guard: materialising attachment-only nodes must not reintroduce the phantom-root
+  // bug. A minted (synthetic) rootRequestKey with an orphan attachment adds a second parentless
+  // node; the tree must still root at the real recorded instance, not the empty synthetic phantom.
+  const edges: LineageEdge[] = [
+    { rootRequestKey: "req-UUID", instanceKey: "inst-root", edgeType: "weak" },
+    { rootRequestKey: "req-UUID", instanceKey: "inst-child", causedByInstanceKey: "inst-root", edgeType: "strong" },
+  ];
+  const tree = buildLineageTree("req-UUID", edges, [{ nodeKey: "inst-orphan", kind: "note", ref: "n-1" }]);
+  assert.equal(tree.root.instanceKey, "inst-root");
+  assert.deepEqual(tree.root.children.map((c) => c.instanceKey), ["inst-child"]);
+  assert.equal(tree.nodes.some((n) => n.instanceKey === "req-UUID"), false);
+  // The orphan attachment is preserved (unattached), and the synthetic key never leaks as a node.
+  const orphan = tree.nodes.find((n) => n.instanceKey === "inst-orphan");
+  assert.ok(orphan, "orphan attachment node must be preserved");
+  assert.deepEqual(orphan.attachments, [{ nodeKey: "inst-orphan", kind: "note", ref: "n-1" }]);
+});
+
+test("buildLineageTree roots a MINTED (synthetic) rootRequestKey at the parentless instance, not a phantom", () => {
+  // A genuine top-level request MINTS a UUID `rootRequestKey` that is NOT any instance key
+  // (`deriveLineage` with no ambient job). The real root instance carries that key with no
+  // cause, and its descendants hang off the instance. The tree must be rooted at the real
+  // parentless instance — never at an empty synthetic node while the real root floats detached.
+  const edges: LineageEdge[] = [
+    { rootRequestKey: "req-UUID", instanceKey: "inst-root", edgeType: "weak" },
+    { rootRequestKey: "req-UUID", instanceKey: "inst-child", causedByInstanceKey: "inst-root", edgeType: "weak" },
+    { rootRequestKey: "req-UUID", instanceKey: "inst-grand", causedByInstanceKey: "inst-child", edgeType: "strong" },
+  ];
+  const tree = buildLineageTree("req-UUID", edges);
+  assert.equal(tree.root.instanceKey, "inst-root");
+  assert.deepEqual(tree.root.children.map((c) => c.instanceKey), ["inst-child"]);
+  assert.deepEqual(tree.root.children[0].children.map((c) => c.instanceKey), ["inst-grand"]);
+  // The synthetic correlation key must not leak into the flat view as a phantom node.
+  assert.equal(tree.nodes.some((n) => n.instanceKey === "req-UUID"), false);
+  assert.equal(tree.nodes.length, 3);
+});
+
+test("buildLineageTree keeps a lone real root (parentless, no descendants) as the tree root", () => {
+  // Guard the fallback boundary: a real root instance whose key IS the rootRequestKey and which
+  // has no descendants must stay the root — it must not be mistaken for a replaceable phantom.
+  const edges: LineageEdge[] = [{ rootRequestKey: "root", instanceKey: "root", edgeType: "weak" }];
+  const tree = buildLineageTree("root", edges);
+  assert.equal(tree.root.instanceKey, "root");
+  assert.equal(tree.root.children.length, 0);
+  assert.equal(tree.nodes.length, 1);
+});
+
+test("buildLineageTree keeps the synthetic root when no edges exist at all", () => {
+  const tree = buildLineageTree("req-UUID", []);
+  assert.equal(tree.root.instanceKey, "req-UUID");
+  assert.equal(tree.root.children.length, 0);
+  assert.deepEqual(tree.nodes.map((n) => n.instanceKey), ["req-UUID"]);
+});