@desplega.ai/agent-swarm 1.129.0 → 1.131.0

package/src/apps/row-store.ts

@@ -14,12 +14,28 @@ export type AppRow = {
   updatedAt: string;
   createdBy?: string;
   updatedBy?: string;
+  /** Sync provenance — present only on rows a source owns. */
+  source?: string;
+  syncedAt?: string;
+  stale?: boolean;
 } & Record<string, unknown>;
 
+export type AppRowEnvelope = { source: string; syncedAt: string; stale: boolean };
+
 export interface AppRowWriteOptions {
   skipUpdatedAt?: boolean;
-  /** Stable id of the acting principal (`user:<id>`, `agent:<id>`, `operator`) for row provenance. */
+  /**
+   * Stable id of the acting principal (`user:<id>`, `agent:<id>`, `operator`,
+   * or `sync:<source>` for engine writes) for row provenance.
+   */
   actor?: string;
+  /**
+   * Sync-engine escape hatch. External write paths leave this false so
+   * source-bound and join-key columns stay read-only projections.
+   */
+  allowSourceManaged?: boolean;
+  /** Row provenance stamped after value preparation; honoured only with `allowSourceManaged`. */
+  envelope?: AppRowEnvelope;
 }
 
 export class AppRowValidationError extends Error {
@@ -92,18 +108,50 @@ function validValue(column: ColumnDef, value: unknown): boolean {
   return typeof value === "string" && Boolean(column.enum?.includes(value));
 }
 
+function joinKeyColumnNames(definition: ModelDef): Set<string> {
+  const names = new Set<string>();
+  for (const source of Object.values(definition.sources ?? {})) names.add(source.joinKey);
+  return names;
+}
+
+/** Row provenance to stamp, when the caller is the sync engine. */
+function sourceEnvelope(options: AppRowWriteOptions): AppRowEnvelope | undefined {
+  return options.allowSourceManaged === true ? options.envelope : undefined;
+}
+
 function prepareValues(
   definition: ModelDef,
   values: Record<string, unknown>,
   mode: "create" | "patch",
+  options: AppRowWriteOptions = {},
 ): Record<string, unknown> {
   const issues: AppValidationIssue[] = [];
   const prepared: Record<string, unknown> = { ...values };
+  const sourceManaged = options.allowSourceManaged === true;
+  if (sourceManaged && !options.envelope) {
+    // Without provenance the privileged write would produce a row
+    // indistinguishable from a user-owned one.
+    throw new Error("allowSourceManaged writes must carry an envelope");
+  }
+  const joinKeys = sourceManaged ? new Set<string>() : joinKeyColumnNames(definition);
 
   for (const name of Object.keys(values)) {
     const column = Object.hasOwn(definition.columns, name) ? definition.columns[name] : undefined;
     if (!column || column.hidden === true) {
       issues.push({ path: `values.${name}`, message: `unknown or hidden column "${name}"` });
+      continue;
+    }
+    if (sourceManaged) continue;
+    if (column.source) {
+      issues.push({
+        path: `values.${name}`,
+        message: `column is a read-only projection from source "${column.source.of}"; mutate it via the source or a sync refresh`,
+      });
+    } else if (joinKeys.has(name)) {
+      issues.push({
+        path: `values.${name}`,
+        message: "column is the sync join key and is managed by the sync engine",
+      });
     }
   }
 
@@ -112,6 +160,9 @@ function prepareValues(
       if (column.hidden === true) continue;
       if (!Object.hasOwn(prepared, name) && column.default !== undefined)
         prepared[name] = column.default;
+      // A source pull supplies only the fields the record carried; a bound
+      // column with no projected value is the source's business, not the row's.
+      if (sourceManaged && column.source) continue;
       if (
         column.required === true &&
         (!Object.hasOwn(prepared, name) || prepared[name] === undefined || prepared[name] === null)
@@ -186,17 +237,19 @@ function createRowUnlocked(
   model: string,
   definition: ModelDef,
   prepared: Record<string, unknown>,
-  actor?: string,
+  options: AppRowWriteOptions,
 ): AppRow {
   const issuedMs = Math.max(Date.now(), lastCreatedAtMs + 1);
   lastCreatedAtMs = issuedMs;
   const now = new Date(issuedMs).toISOString();
+  const actor = options.actor;
   return writeRow(appId, model, definition, {
     id: crypto.randomUUID(),
     createdAt: now,
     updatedAt: now,
     ...(actor !== undefined ? { createdBy: actor, updatedBy: actor } : {}),
     ...prepared,
+    ...sourceEnvelope(options),
   });
 }
 
@@ -208,9 +261,9 @@ export function createAppRowUnlocked(
   values: Record<string, unknown>,
   options: AppRowWriteOptions = {},
 ): AppRow {
-  const prepared = prepareValues(definition, values, "create");
+  const prepared = prepareValues(definition, values, "create", options);
   if (!appExists(appId)) throw new AppRowAppNotFoundError(appId);
-  return createRowUnlocked(appId, model, definition, prepared, options.actor);
+  return createRowUnlocked(appId, model, definition, prepared, options);
 }
 
 export function createAppRow(
@@ -223,8 +276,8 @@ export function createAppRow(
   return withMutationLock(appId, model, () => {
     const currentDefinition = currentModelDefinition(appId, model);
     if (!currentDefinition) throw new AppRowAppNotFoundError(appId);
-    const prepared = prepareValues(currentDefinition, values, "create");
-    return createRowUnlocked(appId, model, currentDefinition, prepared, options.actor);
+    const prepared = prepareValues(currentDefinition, values, "create", options);
+    return createRowUnlocked(appId, model, currentDefinition, prepared, options);
   });
 }
 
@@ -238,9 +291,11 @@ export function createAppRows(
   return withMutationLock(appId, model, () => {
     const currentDefinition = currentModelDefinition(appId, model);
     if (!currentDefinition) throw new AppRowAppNotFoundError(appId);
-    const prepared = rows.map((values) => prepareValues(currentDefinition, values, "create"));
+    const prepared = rows.map((values) =>
+      prepareValues(currentDefinition, values, "create", options),
+    );
     return prepared.map((values) =>
-      createRowUnlocked(appId, model, currentDefinition, values, options.actor),
+      createRowUnlocked(appId, model, currentDefinition, values, options),
     );
   });
 }
@@ -299,7 +354,7 @@ export function patchAppRow(
   return withMutationLock(appId, model, () => {
     const currentDefinition = currentModelDefinition(appId, model);
     if (!currentDefinition) throw new AppRowAppNotFoundError(appId);
-    const prepared = prepareValues(currentDefinition, values, "patch");
+    const prepared = prepareValues(currentDefinition, values, "patch", options);
     return patchPreparedRowUnlocked(appId, model, currentDefinition, rowId, prepared, options);
   });
 }
@@ -332,6 +387,7 @@ function patchPreparedRowUnlocked(
       updated[name] = value;
     }
   }
+  Object.assign(updated, sourceEnvelope(options));
   const newKeys = new Set(indexKeys(model, definition, updated));
   const namespace = appsNamespace(appId);
   for (const key of oldKeys) if (!newKeys.has(key)) deleteKv(namespace, key);
@@ -351,7 +407,7 @@ export function patchAppRowUnlocked(
   values: Record<string, unknown>,
   options: AppRowWriteOptions = {},
 ): AppRow | null {
-  const prepared = prepareValues(definition, values, "patch");
+  const prepared = prepareValues(definition, values, "patch", options);
   return patchPreparedRowUnlocked(appId, model, definition, rowId, prepared, options);
 }
 

package/src/apps/schema-migrate.ts

@@ -10,6 +10,7 @@ import {
   isIso8601Date,
   type ModelDef,
   parseAppDefinition,
+  type SourceDef,
   SYSTEM_COLUMN_KINDS,
 } from "./definition";
 import { upgradeAppDefinition } from "./format-upgrades";
@@ -20,6 +21,7 @@ import {
   withMutationLock,
   writeAppRowForMigrationUnlocked,
 } from "./row-store";
+import { invalidateChangedSyncStatus } from "./sync";
 
 const MigrationValueSchema = z.union([z.string(), z.number(), z.boolean()]);
 
@@ -49,6 +51,7 @@ export const AppMigrationReportSchema = z.object({
   elsed: z.number().int().nonnegative(),
   purgedValues: z.number().int().nonnegative(),
   idxRebuilt: z.number().int().nonnegative(),
+  detachedRows: z.number().int().nonnegative(),
   orphanFields: z.array(z.string()),
   userConfigChanged: z.array(z.string()),
 });
@@ -61,6 +64,7 @@ export const AppMigrationReportOutputSchema = z.looseObject({
   elsed: z.number().optional(),
   purgedValues: z.number().optional(),
   idxRebuilt: z.number().optional(),
+  detachedRows: z.number().optional(),
   orphanFields: z.array(z.string()).optional(),
   userConfigChanged: z.array(z.string()).optional(),
 });
@@ -102,6 +106,7 @@ const EMPTY_REPORT: AppMigrationReport = {
   elsed: 0,
   purgedValues: 0,
   idxRebuilt: 0,
+  detachedRows: 0,
   orphanFields: [],
   userConfigChanged: [],
 };
@@ -637,6 +642,67 @@ function applyDirective(
   }
 }
 
+function ownSource(model: ModelDef | undefined, sourceName: string): SourceDef | undefined {
+  const sources = model?.sources;
+  return sources && Object.hasOwn(sources, sourceName) ? sources[sourceName] : undefined;
+}
+
+/**
+ * Source lifecycle classification. Adding a source, and editing its
+ * args/config/connection/scriptId, are free. Join-key identity is immutable and
+ * a connector swap is refused while the source owns rows — both would silently
+ * re-key or orphan the rows the next pass reconciles against. Removing a source
+ * detaches its rows instead of destroying them (Invariant I4): the values stay,
+ * the `source`/`syncedAt`/`stale` envelope is stripped, and the count is
+ * reported. Detached rows ride the plan's ordinary changed-row write, which
+ * never touches `updatedAt`/`updatedBy` — a detach is not a data edit.
+ */
+function planSources(
+  modelName: string,
+  oldModel: ModelDef | undefined,
+  nextModel: ModelDef | undefined,
+  rows: AppRow[],
+  report: AppMigrationReport,
+  issues: AppValidationIssue[],
+): void {
+  const sourceNames = new Set([
+    ...Object.keys(oldModel?.sources ?? {}),
+    ...Object.keys(nextModel?.sources ?? {}),
+  ]);
+  for (const sourceName of [...sourceNames].sort()) {
+    const oldSource = ownSource(oldModel, sourceName);
+    // A source that did not exist before owns no rows, so every shape it
+    // declares is free (Phase-1 validation still gets its say).
+    if (!oldSource) continue;
+    const nextSource = ownSource(nextModel, sourceName);
+    const path = `models.${modelName}.sources.${sourceName}`;
+    const owned = rows.filter((row) => row.source === sourceName);
+
+    if (!nextSource) {
+      for (const row of owned) {
+        delete row.source;
+        delete row.syncedAt;
+        delete row.stale;
+        report.detachedRows += 1;
+      }
+      continue;
+    }
+
+    if (nextSource.joinKey !== oldSource.joinKey) {
+      issues.push({
+        path: `${path}.joinKey`,
+        message: "join key is immutable; remove the source and add it again",
+      });
+    }
+    if (nextSource.connector !== oldSource.connector && owned.length > 0) {
+      issues.push({
+        path: `${path}.connector`,
+        message: `connector change would orphan ${owned.length} row(s) this source owns; remove the source and add it again`,
+      });
+    }
+  }
+}
+
 function planModel(
   appId: string,
   modelName: string,
@@ -662,6 +728,8 @@ function planModel(
     });
   }
 
+  planSources(modelName, oldModel, nextModel, rows, report, issues);
+
   for (const columnName of changedColumns) {
     const oldColumn = ownColumn(oldModel, columnName);
     const nextColumn = ownColumn(nextModel, columnName);
@@ -696,6 +764,28 @@ function planModel(
       continue;
     }
 
+    // Binding a column that already holds data hands it to the source: the next
+    // pass projects over every value it did not write. A REBIND (source.of
+    // moving to a different source) is the same hazard — the old source stops
+    // projecting the column, the new one only reconciles its own rows, and the
+    // stranded values stay read-only forever. Adding the binding on a fresh
+    // (or emptied) column is the supported path.
+    if (
+      nextColumn.source !== undefined &&
+      (oldColumn?.source === undefined || oldColumn.source.of !== nextColumn.source.of)
+    ) {
+      const populated = rows.filter(
+        (row) => Object.hasOwn(row, columnName) && row[columnName] !== null,
+      ).length;
+      if (populated > 0) {
+        issues.push({
+          path: `${path}.source`,
+          message: `binding an existing column would let the next pass overwrite ${populated} row(s) of existing data; hide or purge the column and add it bound instead`,
+        });
+        continue;
+      }
+    }
+
     if (exactUnhide && nextColumn.required === true && (!directive || !("set" in directive))) {
       const missing = rows.filter(
         (row) => !Object.hasOwn(row, columnName) || row[columnName] === null,
@@ -973,7 +1063,11 @@ export async function migrateAppSchema<T>(input: {
           );
         }
       }
-      return input.writeDefinition();
+      const written = input.writeDefinition();
+      // The stored per-pair sync status describes the OLD configuration:
+      // presenting it for a changed pair would claim a pass that never ran.
+      invalidateChangedSyncStatus(input.appId, input.previousDefinition, input.nextDefinition);
+      return written;
     });
     return { result: transaction(), migration: plan.report };
   });

package/src/apps/store.ts

@@ -1,3 +1,4 @@
+import { defaultAssetKey } from "../assets/key";
 import { getDb } from "../be/db";
 import type { AppDefinition, AppValidationIssue } from "./definition";
 import { AppDefinitionSchema, appDefinitionIssues } from "./definition";
@@ -105,12 +106,20 @@ export function createApp(input: {
   const id = input.id ?? crypto.randomUUID();
   const now = nextTimestamp();
   const row = getDb()
-    .prepare<AppDbRow, [string, string, string | null, string, string, string]>(
-      `INSERT INTO apps (id, name, description, definition, created_at, updated_at)
-       VALUES (?, ?, ?, ?, ?, ?)
+    .prepare<AppDbRow, [string, string, string | null, string, string, string, string]>(
+      `INSERT INTO apps (id, name, description, definition, created_at, updated_at, "key")
+       VALUES (?, ?, ?, ?, ?, ?, ?)
        RETURNING id, name, description, definition, created_at, updated_at`,
     )
-    .get(id, input.name, input.description ?? null, encodeDefinition(input.definition), now, now);
+    .get(
+      id,
+      input.name,
+      input.description ?? null,
+      encodeDefinition(input.definition),
+      now,
+      now,
+      defaultAssetKey("app", id),
+    );
   if (!row) throw new Error("Failed to create app");
   return decodeApp(row);
 }

package/src/apps/sync-run-as.ts

@@ -0,0 +1,18 @@
+import { getLeadAgent } from "../be/db";
+import { getSavedScriptOwnerAgentId } from "../be/scripts/run-saved";
+import type { ScriptRecord } from "../types";
+
+/**
+ * Whose credential and connection bindings a source pull runs with.
+ *
+ * The script owner when there is one; otherwise the lead, because seeded
+ * catalog scripts are owner-less (`scope:"global"`, no `createdByAgentId`) and
+ * the whole catalog-source design depends on them being runnable. The literal
+ * fallback only applies to a swarm with no lead registered yet.
+ *
+ * Definition validation and the sync engine both resolve through here so they
+ * can never disagree about whose connections count.
+ */
+export function resolveSyncRunAs(script: ScriptRecord): string {
+  return getSavedScriptOwnerAgentId(script) ?? getLeadAgent()?.id ?? "app-sync";
+}