@agent-native/core 0.92.12 → 0.93.0

package/corpus/templates/content/actions/add-content-database-source-field-property.ts

@@ -1,6 +1,6 @@
 import { defineAction } from "@agent-native/core";
 import { assertAccess } from "@agent-native/core/sharing";
-import { and, eq, sql } from "drizzle-orm";
+import { and, eq, isNull, sql } from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
@@ -22,12 +22,21 @@ import {
   type DocumentPropertyType,
 } from "../shared/properties.js";
 import { chunks } from "./_batch-utils.js";
+import { readBuilderCmsContentEntries } from "./_builder-cms-read-client.js";
+import {
+  builderCmsQualifiedId,
+  type BuilderCmsSourceEntry,
+} from "./_builder-cms-source-adapter.js";
 import {
   resolveDatabaseForSourceMutation,
   serializeSourceField,
 } from "./_database-source-utils.js";
 import { nanoid } from "./_property-utils.js";
 
+const BUILDER_FIELD_REFRESH_MINIMUM_LIMIT = 500;
+
+type SourceRow = typeof schema.contentDatabaseSourceRows.$inferSelect;
+
 function parseSourceValues(
   value: string | null | undefined,
 ): Record<string, DocumentPropertyValue> {
@@ -42,6 +51,60 @@ function parseSourceValues(
   }
 }
 
+function hasSourceFieldValue(
+  row: Pick<SourceRow, "sourceValuesJson">,
+  sourceFieldKey: string,
+) {
+  return Object.prototype.hasOwnProperty.call(
+    parseSourceValues(row.sourceValuesJson),
+    sourceFieldKey,
+  );
+}
+
+function mergeBuilderFieldIntoSourceRows(args: {
+  rows: SourceRow[];
+  entries: BuilderCmsSourceEntry[];
+  sourceTable: string;
+  sourceFieldKey: string;
+  now: string;
+}) {
+  const entriesById = new Map(args.entries.map((entry) => [entry.id, entry]));
+  const entriesByQualifiedId = new Map(
+    args.entries.map((entry) => [
+      builderCmsQualifiedId({
+        sourceTable: args.sourceTable,
+        entryId: entry.id,
+      }),
+      entry,
+    ]),
+  );
+  let missingRows = 0;
+  let matchedMissingRows = 0;
+  const rows = args.rows.map((row) => {
+    if (hasSourceFieldValue(row, args.sourceFieldKey)) return row;
+    missingRows += 1;
+    const entry =
+      entriesById.get(row.sourceRowId) ??
+      entriesByQualifiedId.get(row.sourceQualifiedId);
+    if (!entry) return row;
+    matchedMissingRows += 1;
+    return {
+      ...row,
+      sourceValuesJson: JSON.stringify({
+        ...parseSourceValues(row.sourceValuesJson),
+        [args.sourceFieldKey]: Object.prototype.hasOwnProperty.call(
+          entry.sourceValues,
+          args.sourceFieldKey,
+        )
+          ? entry.sourceValues[args.sourceFieldKey]
+          : null,
+      }),
+      updatedAt: args.now,
+    };
+  });
+  return { rows, missingRows, matchedMissingRows };
+}
+
 export function sourceFieldPropertyValuesFromRows(
   rows: Array<{
     databaseItemId: string;
@@ -305,24 +368,6 @@ export default defineAction({
       throw new Error("The title source field is already mapped to Name.");
     }
 
-    const now = new Date().toISOString();
-    const visibility = normalizePropertyVisibility(undefined);
-    const [maxPos] = await db
-      .select({
-        max: sql<number>`COALESCE(MAX(position), -1)`,
-      })
-      .from(schema.documentPropertyDefinitions)
-      .where(
-        and(
-          eq(
-            schema.documentPropertyDefinitions.ownerEmail,
-            database.ownerEmail,
-          ),
-          eq(schema.documentPropertyDefinitions.databaseId, database.id),
-        ),
-      );
-    const propertyId = nanoid();
-
     // A federated secondary source's rows have no local document (they join by
     // canonical key), so we don't materialize their values into
     // documentPropertyValues — the read path overlays them per row at query
@@ -338,113 +383,266 @@ export default defineAction({
     }
     const isSecondary = federationRole === "secondary";
 
-    const sourceRows = isSecondary
+    const initialSourceRows = isSecondary
       ? []
       : await db
-          .select({
-            databaseItemId: schema.contentDatabaseSourceRows.databaseItemId,
-            documentId: schema.contentDatabaseSourceRows.documentId,
-            sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
-          })
+          .select()
           .from(schema.contentDatabaseSourceRows)
           .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id));
-    const builderMetadata = builderMetadataForSourceField({
-      sourceFieldKey: field.sourceFieldKey,
-      sourceMetadataJson: source.metadataJson,
-    });
-    const type = propertyTypeForSourceField(
-      field.sourceFieldType,
-      builderMetadata,
-    );
-    const options = sourceFieldPropertyOptions({
-      type,
-      metadata: builderMetadata,
-      rows: sourceRows,
-      sourceFieldKey: field.sourceFieldKey,
-    });
+    const shouldRefreshBuilderField =
+      !isSecondary &&
+      source.sourceType === "builder-cms" &&
+      initialSourceRows.some(
+        (row) => !hasSourceFieldValue(row, field.sourceFieldKey),
+      );
+    let builderEntries: BuilderCmsSourceEntry[] | null = null;
+    if (shouldRefreshBuilderField) {
+      // Read before writing so a Builder outage cannot leave behind a cleanly
+      // reported but empty property. Start at zero even when the source's
+      // broader refresh is incremental: the stored rows we are backfilling
+      // begin with that first page.
+      const builderRead = await readBuilderCmsContentEntries({
+        model: source.sourceTable,
+        fieldPaths: [field.sourceFieldKey],
+        limit: Math.max(
+          BUILDER_FIELD_REFRESH_MINIMUM_LIMIT,
+          initialSourceRows.length,
+        ),
+        offset: 0,
+      });
+      if (builderRead.state !== "live") {
+        throw new Error(
+          builderRead.message ??
+            "Builder CMS could not refresh this field. No property was created.",
+        );
+      }
+      if (initialSourceRows.length > 0 && builderRead.entries.length === 0) {
+        throw new Error(
+          "Builder CMS returned no entries for this source. No property was created; refresh the source and try again.",
+        );
+      }
+      builderEntries = builderRead.entries;
+    }
 
-    await db.insert(schema.documentPropertyDefinitions).values({
-      id: propertyId,
-      ownerEmail: database.ownerEmail,
-      orgId: database.orgId ?? null,
-      databaseId: database.id,
-      name: field.sourceFieldLabel,
-      type,
-      visibility,
-      optionsJson: serializePropertyOptions(options),
-      position: (maxPos?.max ?? -1) + 1,
-      createdAt: now,
-      updatedAt: now,
-    });
+    // Keep the local snapshot, property definition, mapping, and materialized
+    // values atomic. Re-read rows after the provider request so a concurrent
+    // source refresh cannot be overwritten by the older pre-request snapshot.
+    return await db.transaction(async (tx) => {
+      const now = new Date().toISOString();
+      const visibility = normalizePropertyVisibility(undefined);
+      const propertyId = nanoid();
+      const [currentField] = await tx
+        .select()
+        .from(schema.contentDatabaseSourceFields)
+        .where(eq(schema.contentDatabaseSourceFields.id, field.id));
+      if (!currentField) throw new Error("Source field not found.");
+      if (currentField.propertyId) {
+        throw new Error("Source field is already mapped to a property.");
+      }
+      if (currentField.mappingType === "title") {
+        throw new Error("The title source field is already mapped to Name.");
+      }
+      if (currentField.sourceId !== source.id) {
+        throw new Error("Source field does not belong to this database.");
+      }
+      const [currentSource] = await tx
+        .select()
+        .from(schema.contentDatabaseSources)
+        .where(
+          and(
+            eq(schema.contentDatabaseSources.id, source.id),
+            eq(schema.contentDatabaseSources.databaseId, database.id),
+          ),
+        );
+      if (!currentSource) {
+        throw new Error("Source field does not belong to this database.");
+      }
 
-    await db
-      .update(schema.contentDatabaseSourceFields)
-      .set({
-        propertyId,
-        localFieldKey: propertyId,
-        mappingType: "property",
-        updatedAt: now,
-      })
-      .where(eq(schema.contentDatabaseSourceFields.id, field.id));
-
-    await db
-      .update(schema.contentDatabaseSources)
-      .set({ updatedAt: now })
-      .where(eq(schema.contentDatabaseSources.id, source.id));
-
-    const itemValues = sourceFieldPropertyValuesFromRows(
-      sourceRows,
-      field.sourceFieldKey,
-      type,
-      options,
-    );
-    if (itemValues.length > 0) {
-      for (const chunk of chunks(itemValues, 200)) {
-        await db.insert(schema.documentPropertyValues).values(
-          chunk.map((row) => ({
-            id: nanoid(),
-            ownerEmail: database.ownerEmail,
-            documentId: row.documentId,
-            propertyId,
-            valueJson: serializePropertyValue(row.value),
-            createdAt: now,
-            updatedAt: now,
-          })),
+      let sourceRows = isSecondary
+        ? []
+        : await tx
+            .select()
+            .from(schema.contentDatabaseSourceRows)
+            .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id));
+      if (builderEntries) {
+        const merged = mergeBuilderFieldIntoSourceRows({
+          rows: sourceRows,
+          entries: builderEntries,
+          sourceTable: source.sourceTable,
+          sourceFieldKey: currentField.sourceFieldKey,
+          now,
+        });
+        if (merged.matchedMissingRows < merged.missingRows) {
+          throw new Error(
+            "Builder CMS did not return entries for every stored source row. No property was created; refresh the source and try again.",
+          );
+        }
+        for (let index = 0; index < merged.rows.length; index += 1) {
+          const currentRow = sourceRows[index];
+          const mergedRow = merged.rows[index];
+          if (
+            !currentRow ||
+            !mergedRow ||
+            currentRow.sourceValuesJson === mergedRow.sourceValuesJson
+          ) {
+            continue;
+          }
+          const [updatedRow] = await tx
+            .update(schema.contentDatabaseSourceRows)
+            .set({
+              sourceValuesJson: mergedRow.sourceValuesJson,
+              updatedAt: now,
+            })
+            .where(
+              and(
+                eq(schema.contentDatabaseSourceRows.id, currentRow.id),
+                eq(
+                  schema.contentDatabaseSourceRows.sourceValuesJson,
+                  currentRow.sourceValuesJson,
+                ),
+              ),
+            )
+            .returning({ id: schema.contentDatabaseSourceRows.id });
+          if (!updatedRow) {
+            throw new Error(
+              "The Builder source changed while adding this property. No property was created; try again.",
+            );
+          }
+        }
+        sourceRows = merged.rows;
+      } else if (
+        source.sourceType === "builder-cms" &&
+        sourceRows.some(
+          (row) => !hasSourceFieldValue(row, currentField.sourceFieldKey),
+        )
+      ) {
+        throw new Error(
+          "The Builder source changed while adding this property. No property was created; try again.",
         );
       }
-    }
 
-    const sourceField = serializeSourceField(
-      {
-        ...field,
-        propertyId,
-        localFieldKey: propertyId,
-        mappingType: "property",
+      const builderMetadata = builderMetadataForSourceField({
+        sourceFieldKey: currentField.sourceFieldKey,
+        sourceMetadataJson: currentSource.metadataJson,
+      });
+      const type = propertyTypeForSourceField(
+        currentField.sourceFieldType,
+        builderMetadata,
+      );
+      const options = sourceFieldPropertyOptions({
+        type,
+        metadata: builderMetadata,
+        rows: sourceRows,
+        sourceFieldKey: currentField.sourceFieldKey,
+      });
+      const [maxPos] = await tx
+        .select({
+          max: sql<number>`COALESCE(MAX(position), -1)`,
+        })
+        .from(schema.documentPropertyDefinitions)
+        .where(
+          and(
+            eq(
+              schema.documentPropertyDefinitions.ownerEmail,
+              database.ownerEmail,
+            ),
+            eq(schema.documentPropertyDefinitions.databaseId, database.id),
+          ),
+        );
+      const position = (maxPos?.max ?? -1) + 1;
+
+      await tx.insert(schema.documentPropertyDefinitions).values({
+        id: propertyId,
+        ownerEmail: database.ownerEmail,
+        orgId: database.orgId ?? null,
+        databaseId: database.id,
+        name: currentField.sourceFieldLabel,
+        type,
+        visibility,
+        optionsJson: serializePropertyOptions(options),
+        position,
+        createdAt: now,
         updatedAt: now,
-      },
-      field.sourceFieldLabel,
-    );
+      });
 
-    return {
-      databaseId: database.id,
-      documentId: database.documentId,
-      property: {
-        definition: {
-          id: propertyId,
-          databaseId: database.id,
-          name: field.sourceFieldLabel,
-          type,
-          visibility,
-          options,
-          position: (maxPos?.max ?? -1) + 1,
-          createdAt: now,
+      const [mappedField] = await tx
+        .update(schema.contentDatabaseSourceFields)
+        .set({
+          propertyId,
+          localFieldKey: propertyId,
+          mappingType: "property",
+          updatedAt: now,
+        })
+        .where(
+          and(
+            eq(schema.contentDatabaseSourceFields.id, currentField.id),
+            isNull(schema.contentDatabaseSourceFields.propertyId),
+          ),
+        )
+        .returning({ id: schema.contentDatabaseSourceFields.id });
+      if (!mappedField) {
+        throw new Error("Source field is already mapped to a property.");
+      }
+
+      await tx
+        .update(schema.contentDatabaseSources)
+        .set({ updatedAt: now })
+        .where(eq(schema.contentDatabaseSources.id, currentSource.id));
+
+      const itemValues = sourceFieldPropertyValuesFromRows(
+        sourceRows,
+        currentField.sourceFieldKey,
+        type,
+        options,
+      );
+      if (itemValues.length > 0) {
+        for (const chunk of chunks(itemValues, 200)) {
+          await tx.insert(schema.documentPropertyValues).values(
+            chunk.map((row) => ({
+              id: nanoid(),
+              ownerEmail: database.ownerEmail,
+              documentId: row.documentId,
+              propertyId,
+              valueJson: serializePropertyValue(row.value),
+              createdAt: now,
+              updatedAt: now,
+            })),
+          );
+        }
+      }
+
+      const sourceField = serializeSourceField(
+        {
+          ...currentField,
+          propertyId,
+          localFieldKey: propertyId,
+          mappingType: "property",
           updatedAt: now,
         },
-        value: null,
-        editable: true,
-      },
-      sourceField,
-      itemValues,
-    };
+        currentField.sourceFieldLabel,
+      );
+
+      return {
+        databaseId: database.id,
+        documentId: database.documentId,
+        property: {
+          definition: {
+            id: propertyId,
+            databaseId: database.id,
+            name: currentField.sourceFieldLabel,
+            type,
+            visibility,
+            options,
+            position,
+            createdAt: now,
+            updatedAt: now,
+          },
+          value: null,
+          editable: true,
+        },
+        sourceField,
+        itemValues,
+      };
+    });
   },
 });