@agent-native/core 0.80.5 → 0.80.6

package/corpus/templates/content/actions/_database-utils.ts

@@ -19,6 +19,8 @@ import {
   serializeDatabase,
 } from "./_property-utils.js";
 
+export const CONTENT_DATABASE_MAX_READ_LIMIT = 5_000;
+
 function canManageRole(role: string) {
   return role === "owner" || role === "admin";
 }
@@ -78,7 +80,10 @@ export function normalizeContentDatabasePageOptions(options: {
 }) {
   const limit =
     typeof options.limit === "number" && Number.isFinite(options.limit)
-      ? Math.max(1, Math.min(Math.floor(options.limit), 500))
+      ? Math.max(
+          1,
+          Math.min(Math.floor(options.limit), CONTENT_DATABASE_MAX_READ_LIMIT),
+        )
       : null;
   const offset =
     typeof options.offset === "number" && Number.isFinite(options.offset)
@@ -181,21 +186,19 @@ export async function getContentDatabaseResponse(
   const serializedDocumentIds = new Set(
     serializedItems.map((item) => item.document.id),
   );
-  // When paginating, the *primary* source's rows are document-backed, so we can
-  // scope them to the visible page. Secondary rows join by canonical key (no
-  // document), so they're left intact — only matched ones overlay anyway.
+  // When paginating, scope every DOCUMENT-BACKED source's rows to the visible
+  // page — that's the primary AND any row-union secondary (each row maps to a
+  // real document). Federated join rows carry no document (empty documentId),
+  // so they're kept intact — only matched ones overlay anyway.
   const pagedSources =
     limit !== null
-      ? sources.map((source, index) =>
-          index === 0
-            ? {
-                ...source,
-                rows: source.rows.filter((row) =>
-                  serializedDocumentIds.has(row.documentId),
-                ),
-              }
-            : source,
-        )
+      ? sources.map((source) => ({
+          ...source,
+          rows: source.rows.filter(
+            (row) =>
+              !row.documentId || serializedDocumentIds.has(row.documentId),
+          ),
+        }))
       : sources;
   const pagedPrimary = pagedSources[0] ?? null;
 

package/corpus/templates/content/actions/_federation-join.ts

@@ -53,6 +53,13 @@ export function federateSources(args: {
   const primaryRowByDocumentId = new Map(
     (primary?.rows ?? []).map((row) => [row.documentId, row]),
   );
+  const rowByDocumentId = new Map<string, ContentDatabaseSourceRow>();
+  for (const source of sources) {
+    for (const row of source.rows) {
+      if (!row.documentId || rowByDocumentId.has(row.documentId)) continue;
+      rowByDocumentId.set(row.documentId, row);
+    }
+  }
 
   // Secondary sources carrying an identity-join config.
   const secondaries = sources
@@ -82,7 +89,9 @@ export function federateSources(args: {
     return items.map((item) => ({
       ...item,
       sourceRecord:
-        primaryRowByDocumentId.get(item.document.id) ?? item.sourceRecord,
+        primaryRowByDocumentId.get(item.document.id) ??
+        rowByDocumentId.get(item.document.id) ??
+        item.sourceRecord,
     }));
   }
 
@@ -112,7 +121,10 @@ export function federateSources(args: {
 
     return {
       ...item,
-      sourceRecord: primaryRow ?? item.sourceRecord,
+      sourceRecord:
+        primaryRow ??
+        rowByDocumentId.get(item.document.id) ??
+        item.sourceRecord,
       canonicalKey,
       sourceOverlays: overlays.length > 0 ? overlays : undefined,
     };

package/corpus/templates/content/actions/_join-suggestion.ts

@@ -36,7 +36,16 @@ export interface JoinSuggestion {
 
 type ValueRow = Record<string, DocumentPropertyValue>;
 
-const KEY_NAME_HINTS = ["url", "slug", "handle", "id", "key", "path", "ref"];
+const KEY_NAME_HINTS = [
+  "url",
+  "slug",
+  "handle",
+  "id",
+  "key",
+  "path",
+  "ref",
+  "author",
+];
 
 function stringValue(value: DocumentPropertyValue): string | null {
   if (typeof value === "string") return value.trim() ? value : null;
@@ -59,6 +68,18 @@ function proposeFormula(field: string, urlLike: boolean): string {
     : `lower(trim({${field}}))`;
 }
 
+function looksLikeBuilderReference(values: string[]): boolean {
+  return values.some((value) => /^[a-z0-9_-]+:[a-z0-9_-]+$/i.test(value));
+}
+
+function proposeFormulas(field: string, values: string[], urlLike: boolean) {
+  const formulas = [proposeFormula(field, urlLike)];
+  if (looksLikeBuilderReference(values)) {
+    formulas.push(`lower(trim(regexreplace({${field}}, "^[^:]+:", "")))`);
+  }
+  return formulas;
+}
+
 function fieldLabel(field: string): string {
   return (
     field
@@ -98,12 +119,11 @@ function rawStringValuesForField(field: string, rows: ValueRow[]) {
   return raws;
 }
 
-function profileField(
+function profileFieldWithFormula(
   field: string,
   raws: string[],
-  urlLike: boolean,
+  formula: string,
 ): FieldProfile {
-  const formula = proposeFormula(field, urlLike);
   const normalizedByRaw = new Map<string, string>();
   const normalizedSet = new Set<string>();
   for (const raw of raws) {
@@ -123,6 +143,19 @@ function jaccard(a: Set<string>, b: Set<string>): number {
   return union === 0 ? 0 : intersection / union;
 }
 
+function intersectionSize(a: Set<string>, b: Set<string>): number {
+  let intersection = 0;
+  for (const value of a) if (b.has(value)) intersection += 1;
+  return intersection;
+}
+
+function lowInformationField(field: string): boolean {
+  const normalized = field.toLowerCase();
+  return /(^|[._-])(published|state|status|sync_state|created|updated|lastupdated|last_updated)([._-]|$)/.test(
+    normalized,
+  );
+}
+
 function nameNudge(primaryField: string, secondaryField: string): number {
   const hinted = (field: string) =>
     KEY_NAME_HINTS.some((hint) => field.toLowerCase().includes(hint));
@@ -164,17 +197,59 @@ export function suggestJoinKey(args: {
     for (const [secondaryField, secondaryRaws] of secondaryRawByField) {
       const pairUrlLike =
         looksLikeUrl(primaryRaws) || looksLikeUrl(secondaryRaws);
-      const primary = profileField(primaryField, primaryRaws, pairUrlLike);
-      const secondary = profileField(
-        secondaryField,
-        secondaryRaws,
+      for (const primaryFormula of proposeFormulas(
+        primaryField,
+        primaryRaws,
         pairUrlLike,
-      );
-      const overlap = jaccard(primary.normalizedSet, secondary.normalizedSet);
-      if (overlap <= 0) continue;
-      const score = overlap + nameNudge(primary.field, secondary.field);
-      if (!best || score > best.score) {
-        best = { primary, secondary, score, overlap };
+      )) {
+        for (const secondaryFormula of proposeFormulas(
+          secondaryField,
+          secondaryRaws,
+          pairUrlLike,
+        )) {
+          const primary = profileFieldWithFormula(
+            primaryField,
+            primaryRaws,
+            primaryFormula,
+          );
+          const secondary = profileFieldWithFormula(
+            secondaryField,
+            secondaryRaws,
+            secondaryFormula,
+          );
+          const overlap = jaccard(
+            primary.normalizedSet,
+            secondary.normalizedSet,
+          );
+          if (overlap <= 0) continue;
+          const sharedCount = intersectionSize(
+            primary.normalizedSet,
+            secondary.normalizedSet,
+          );
+          if (
+            sharedCount < 2 &&
+            (primary.normalizedByRaw.size > 2 ||
+              secondary.normalizedByRaw.size > 2)
+          ) {
+            continue;
+          }
+          const lowInfoPenalty =
+            lowInformationField(primary.field) ||
+            lowInformationField(secondary.field)
+              ? 0.5
+              : 0;
+          const cardinalityNudge =
+            Math.min(primary.normalizedSet.size, secondary.normalizedSet.size) *
+            0.001;
+          const score =
+            overlap +
+            nameNudge(primary.field, secondary.field) +
+            cardinalityNudge -
+            lowInfoPenalty;
+          if (!best || score > best.score) {
+            best = { primary, secondary, score, overlap };
+          }
+        }
       }
     }
   }

package/corpus/templates/content/actions/_local-file-documents.ts

@@ -309,7 +309,8 @@ function upsertFrontmatter(
     return body;
   }
 
-  return `---\n${nextLines.filter(Boolean).join("\n")}\n---\n\n${body}`;
+  const frontmatterBlock = `---\n${nextLines.filter(Boolean).join("\n")}\n---\n`;
+  return body ? `${frontmatterBlock}\n${body}` : frontmatterBlock;
 }
 
 function stripDuplicateTitleHeading(content: string, title: string) {

package/corpus/templates/content/actions/attach-content-database-source.ts

@@ -15,6 +15,8 @@ import {
 } from "./_builder-cms-read-client.js";
 import type { BuilderCmsSourceEntry } from "./_builder-cms-source-adapter.js";
 import {
+  databaseSourceExistsForTable,
+  ensureDatabaseSourceProperty,
   getExistingSource,
   getSourceRows,
   importBuilderCmsEntriesAsDatabaseItems,
@@ -129,10 +131,22 @@ export default defineAction({
       .string()
       .optional()
       .describe("Source table/model name, for example content_items."),
+    relationshipMode: z
+      .enum(["items", "details"])
+      .optional()
+      .describe(
+        "How to attach a second source: items adds more rows; details joins fields onto existing rows.",
+      ),
     join: joinSchema
       .optional()
       .describe(
-        "When adding a SECOND source, the canonical-key join that federates it onto the primary (read-only overlay).",
+        "When relationshipMode is details, the canonical-key join that adds fields onto the primary rows.",
+      ),
+    mode: z
+      .enum(["replace", "add"])
+      .optional()
+      .describe(
+        "Backward-compatible alias: add means relationshipMode items; replace (default) re-links the primary source.",
       ),
     limit: z.coerce.number().int().min(1).max(500).default(100),
     offset: z.coerce.number().int().min(0).default(0),
@@ -160,10 +174,16 @@ export default defineAction({
       await resolveReadableLocalTableSource(sourceTable);
     }
 
-    // Adding a SECOND source: federate it onto the primary on the canonical key
-    // instead of replacing the binding. Read-only overlay — the secondary's
-    // entries are NOT imported as local documents/items.
-    if (args.join && existingSource) {
+    const relationshipMode =
+      args.relationshipMode ?? (args.mode === "add" ? "items" : undefined);
+
+    // Adding a SECOND source as details: relate it onto the primary on the
+    // canonical key. Read-only overlay — the secondary's entries are NOT
+    // imported as local documents/items.
+    if ((relationshipMode === "details" || args.join) && existingSource) {
+      if (!args.join) {
+        throw new Error("Choose a match key before adding source details.");
+      }
       let entries: BuilderCmsSourceEntry[];
       let modelFields: BuilderCmsModelFieldSummary[];
       if (sourceType === "builder-cms") {
@@ -229,6 +249,105 @@ export default defineAction({
       });
     }
 
+    // Adding an ADDITIONAL writable Builder source (row-union): insert a new
+    // source and import its entries as their OWN rows, instead of replacing the
+    // primary. No canonical-key join — each row belongs to exactly one source.
+    if (
+      relationshipMode === "items" &&
+      existingSource &&
+      sourceType === "builder-cms"
+    ) {
+      // Don't add the same collection twice — each "add" starts a fresh source
+      // with no prior rows, so a duplicate attach would re-import duplicate rows.
+      if (await databaseSourceExistsForTable(database.id, sourceTable)) {
+        throw new Error(`"${sourceTable}" is already attached as a source.`);
+      }
+      const additionalRead = await readBuilderCmsContentEntries({
+        model: sourceTable,
+      });
+      const additionalModelFields = await readBuilderCmsModelFields({
+        model: sourceTable,
+      });
+      const additionalSourceId = await insertSecondarySource({
+        database,
+        sourceType,
+        sourceName,
+        sourceTable,
+        now,
+      });
+      // Snapshot existing items BEFORE importing so we can bind the new source
+      // to ONLY the rows it imports — never the primary's existing rows.
+      const beforeSetup = await sourceSetupPayload(database.id);
+      const priorDocumentIds = new Set(
+        beforeSetup.response.items.map((item) => item.document.id),
+      );
+      if (additionalRead.state === "live") {
+        await importBuilderCmsEntriesAsDatabaseItems({
+          database,
+          entries: additionalRead.entries,
+          now,
+          sourceTable,
+          existingSourceRows: [],
+          skipTitleDedup: true,
+        });
+      }
+      const additionalSetup = await sourceSetupPayload(database.id);
+      // Only the items this collection just created — exclude the primary's.
+      const importedItems = additionalSetup.response.items.filter(
+        (item) => !priorDocumentIds.has(item.document.id),
+      );
+      const additionalEntriesByDocumentId =
+        additionalRead.state === "live"
+          ? mapBuilderCmsEntriesToLocalItems({
+              entries: additionalRead.entries,
+              items: importedItems,
+              sourceTable,
+              now,
+              existingRows: [],
+            })
+          : undefined;
+      await seedMockSourceFields({
+        sourceId: additionalSourceId,
+        ownerEmail: database.ownerEmail,
+        sourceType,
+        properties: additionalSetup.properties,
+        builderModelFields: additionalModelFields,
+        builderSampleEntries:
+          additionalRead.state === "live" ? additionalRead.entries : [],
+        now,
+      });
+      await seedMockSourceRows({
+        sourceId: additionalSourceId,
+        ownerEmail: database.ownerEmail,
+        sourceType,
+        sourceTable,
+        items: importedItems,
+        now,
+        builderEntriesByDocumentId: additionalEntriesByDocumentId,
+      });
+      await updateBuilderCmsSourceReadMetadata({
+        sourceId: additionalSourceId,
+        sourceTable,
+        readState: additionalRead.state,
+        entryCount: additionalRead.entries.length,
+        matchedRowCount: additionalEntriesByDocumentId?.size ?? 0,
+        fetchedAt: additionalRead.fetchedAt,
+        now,
+        message: additionalRead.message,
+        syncState: "linked",
+      });
+      await ensureDatabaseSourceProperty({ database, now });
+
+      return getContentDatabaseResponse(database.id, {
+        limit: args.limit,
+        offset: args.offset,
+      });
+    }
+
+    if (relationshipMode === "items" && existingSource) {
+      throw new Error("Only Builder sources can add more items right now.");
+    }
+
     const existingSourceRows = existingSource
       ? await getSourceRows(existingSource.id)
       : [];

package/corpus/templates/content/actions/bind-content-database-source-field.ts

@@ -0,0 +1,256 @@
+import { defineAction } from "@agent-native/core";
+import { assertAccess } from "@agent-native/core/sharing";
+import { and, eq, inArray, ne } from "drizzle-orm";
+import { z } from "zod";
+
+import { getDb, schema } from "../server/db/index.js";
+import type {
+  BindContentDatabaseSourceFieldRequest,
+  ContentDatabaseResponse,
+} from "../shared/api.js";
+import { serializePropertyValue } from "../shared/properties.js";
+import { resolveDatabaseForSourceMutation } from "./_database-source-utils.js";
+import { getContentDatabaseResponse } from "./_database-utils.js";
+import { nanoid } from "./_property-utils.js";
+import {
+  propertyTypeForSourceField,
+  sourceFieldPropertyValuesFromRows,
+} from "./add-content-database-source-field-property.js";
+
+const SOURCE_TAG_PROPERTY_NAME = "Source";
+
+export default defineAction({
+  description:
+    "Bind a source field to an existing database column (row-union per-source field binding), or unbind it. Binding routes the source's per-row values into the shared column; types must be compatible. Pass propertyId: null to unbind.",
+  schema: z.object({
+    databaseId: z.string().optional().describe("Database ID"),
+    documentId: z.string().optional().describe("Database document/page ID"),
+    sourceFieldId: z.string().describe("Source field mapping ID"),
+    propertyId: z
+      .string()
+      .nullable()
+      .describe(
+        "Target column property to bind the field to, or null to unbind.",
+      ),
+  }),
+  run: async (
+    args: BindContentDatabaseSourceFieldRequest,
+  ): Promise<ContentDatabaseResponse> => {
+    const database = await resolveDatabaseForSourceMutation(args);
+    if (!database) throw new Error("Database not found.");
+    await assertAccess("document", database.documentId, "editor");
+
+    const db = getDb();
+    const [field] = await db
+      .select()
+      .from(schema.contentDatabaseSourceFields)
+      .where(eq(schema.contentDatabaseSourceFields.id, args.sourceFieldId));
+    if (!field) throw new Error("Source field not found.");
+
+    const [source] = await db
+      .select()
+      .from(schema.contentDatabaseSources)
+      .where(
+        and(
+          eq(schema.contentDatabaseSources.id, field.sourceId),
+          eq(schema.contentDatabaseSources.databaseId, database.id),
+        ),
+      );
+    if (!source) {
+      throw new Error("Source field does not belong to this database.");
+    }
+    if (field.mappingType === "title") {
+      throw new Error("The title field is bound to Name and can't be rebound.");
+    }
+    if (field.mappingType === "system" || field.writeOwner === "derived") {
+      throw new Error("Integration-managed fields can't be bound to a column.");
+    }
+
+    const now = new Date().toISOString();
+
+    // ── Unbind ────────────────────────────────────────────────────────────
+    if (args.propertyId === null) {
+      if (field.propertyId) {
+        const sourceRows = await db
+          .select({ documentId: schema.contentDatabaseSourceRows.documentId })
+          .from(schema.contentDatabaseSourceRows)
+          .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id));
+        const sourceDocumentIds = sourceRows
+          .map((row) => row.documentId)
+          .filter((id): id is string => Boolean(id));
+        if (sourceDocumentIds.length > 0) {
+          await db
+            .delete(schema.documentPropertyValues)
+            .where(
+              and(
+                eq(schema.documentPropertyValues.propertyId, field.propertyId),
+                inArray(
+                  schema.documentPropertyValues.documentId,
+                  sourceDocumentIds,
+                ),
+              ),
+            );
+        }
+      }
+      await db
+        .update(schema.contentDatabaseSourceFields)
+        .set({
+          propertyId: null,
+          localFieldKey: field.sourceFieldKey,
+          updatedAt: now,
+        })
+        .where(eq(schema.contentDatabaseSourceFields.id, field.id));
+      await db
+        .update(schema.contentDatabaseSources)
+        .set({ updatedAt: now })
+        .where(eq(schema.contentDatabaseSources.id, source.id));
+      return getContentDatabaseResponse(database.id);
+    }
+
+    // ── Bind to an existing column ─────────────────────────────────────────
+    const [property] = await db
+      .select()
+      .from(schema.documentPropertyDefinitions)
+      .where(
+        and(
+          eq(schema.documentPropertyDefinitions.id, args.propertyId),
+          eq(schema.documentPropertyDefinitions.databaseId, database.id),
+        ),
+      );
+    if (!property) {
+      throw new Error("Target column does not belong to this database.");
+    }
+    // The auto-created "Source" tag is internal row-tagging, never a writable
+    // bind target.
+    if (
+      property.name === SOURCE_TAG_PROPERTY_NAME &&
+      property.type === "select"
+    ) {
+      throw new Error(
+        "The Source tag column can't be bound to a source field.",
+      );
+    }
+    // Don't silently repoint a field that's already feeding another column —
+    // that would orphan the old column's materialized values. Require an
+    // explicit unbind first. (Re-binding to the SAME column is an idempotent
+    // refresh and is allowed.)
+    if (field.propertyId && field.propertyId !== property.id) {
+      throw new Error(
+        "This source field is already bound to another column. Unbind it first.",
+      );
+    }
+    // At most one field per source per column: a column reads one value per row,
+    // and a row belongs to one source. Enforce server-side, not just in the UI.
+    const [conflictingField] = await db
+      .select({ id: schema.contentDatabaseSourceFields.id })
+      .from(schema.contentDatabaseSourceFields)
+      .where(
+        and(
+          eq(schema.contentDatabaseSourceFields.sourceId, source.id),
+          eq(schema.contentDatabaseSourceFields.propertyId, property.id),
+          ne(schema.contentDatabaseSourceFields.id, field.id),
+        ),
+      );
+    if (conflictingField) {
+      throw new Error(
+        "This source already feeds this column from another field. Unbind it first.",
+      );
+    }
+    // Only type-compatible fields can share a column. A `text` column is a
+    // permissive target for SCALAR fields; a multi-value (list) field would be
+    // lossily stringified, so it needs a matching list/multi-select column.
+    // Otherwise the field's derived type must equal the column's type.
+    const fieldType = propertyTypeForSourceField(field.sourceFieldType);
+    const fieldIsMultiValue = [
+      "list",
+      "array",
+      "tags",
+      "multi_select",
+    ].includes(field.sourceFieldType.trim().toLowerCase());
+    if (property.type === "text") {
+      if (fieldIsMultiValue) {
+        throw new Error(
+          "A multi-value source field can't be bound to a text column.",
+        );
+      }
+    } else if (property.type !== fieldType) {
+      throw new Error(
+        `Field type "${fieldType}" is not compatible with the "${property.type}" column.`,
+      );
+    }
+
+    await db
+      .update(schema.contentDatabaseSourceFields)
+      .set({
+        propertyId: property.id,
+        localFieldKey: property.id,
+        mappingType: "property",
+        updatedAt: now,
+      })
+      .where(eq(schema.contentDatabaseSourceFields.id, field.id));
+    await db
+      .update(schema.contentDatabaseSources)
+      .set({ updatedAt: now })
+      .where(eq(schema.contentDatabaseSources.id, source.id));
+
+    // Backfill the column with this source's per-row values. A federated
+    // secondary's rows carry no local document (the read path overlays them),
+    // so only materialize for document-backed sources.
+    let federationRole: string | null = null;
+    try {
+      const parsed = JSON.parse(source.metadataJson ?? "{}") as {
+        federation?: { role?: string };
+      };
+      federationRole = parsed.federation?.role ?? null;
+    } catch {
+      federationRole = null;
+    }
+    if (federationRole !== "secondary") {
+      const sourceRows = await db
+        .select()
+        .from(schema.contentDatabaseSourceRows)
+        .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id));
+      const itemValues = sourceFieldPropertyValuesFromRows(
+        sourceRows,
+        field.sourceFieldKey,
+        property.type,
+      );
+      // Clear this column's values for ALL of this source's rows first — not
+      // just the rows that now have a value — so a row whose new bound field is
+      // empty doesn't keep showing a stale/previous value. Then write the
+      // non-empty ones. (This source owns these documents' values for the row-
+      // union, so clearing them is safe.)
+      const sourceDocumentIds = sourceRows
+        .map((row) => row.documentId)
+        .filter((id): id is string => Boolean(id));
+      if (sourceDocumentIds.length > 0) {
+        await db
+          .delete(schema.documentPropertyValues)
+          .where(
+            and(
+              eq(schema.documentPropertyValues.propertyId, property.id),
+              inArray(
+                schema.documentPropertyValues.documentId,
+                sourceDocumentIds,
+              ),
+            ),
+          );
+      }
+      if (itemValues.length > 0) {
+        await db.insert(schema.documentPropertyValues).values(
+          itemValues.map((row) => ({
+            id: nanoid(),
+            ownerEmail: database.ownerEmail,
+            documentId: row.documentId,
+            propertyId: property.id,
+            valueJson: serializePropertyValue(row.value),
+            createdAt: now,
+            updatedAt: now,
+          })),
+        );
+      }
+    }
+
+    return getContentDatabaseResponse(database.id);
+  },
+});