@agent-native/core 0.96.0 → 0.98.0

package/corpus/templates/content/actions/_database-source-utils.ts

@@ -76,6 +76,12 @@ import {
   type ExistingBuilderSourceRowIdentity,
 } from "./_builder-cms-source-adapter.js";
 import { mergeBuilderCmsWriteSettingsIntoJson } from "./_builder-cms-write-settings.js";
+import {
+  databaseItemsPositionScope,
+  documentsPositionScope,
+  propertyDefinitionsPositionScope,
+  withPositionLock,
+} from "./_position-utils.js";
 import { listPropertiesForDatabase, nanoid } from "./_property-utils.js";
 import { isEffectivelyEmptyDocumentContent } from "./update-document.js";
 
@@ -189,7 +195,12 @@ function normalizeWriteOwner(
 function normalizeSourceType(
   value: string | null | undefined,
 ): ContentDatabaseSourceType {
-  if (value === "builder-cms" || value === "local-table") return value;
+  if (
+    value === "builder-cms" ||
+    value === "local-table" ||
+    value === "notion-database"
+  )
+    return value;
   return "mock-local";
 }
 
@@ -279,6 +290,7 @@ function sourceMetadataLabel(
 ) {
   if (sourceType === "builder-cms") return `builder.cms.${sourceTable}`;
   if (sourceType === "local-table") return `local.table.${sourceTable}`;
+  if (sourceType === "notion-database") return `notion.database.${sourceTable}`;
   return `mock-local.${sourceTable}`;
 }
 
@@ -3102,6 +3114,20 @@ export function serializeSourceMetadataRecord(args: {
         args.builderModelFields ?? existingMetadata?.builderModelFields,
     });
   }
+  if (args.sourceType === "notion-database") {
+    return JSON.stringify({
+      primaryKey: "id",
+      titleField: "title",
+      naturalKeyField: null,
+      pushMode: "none",
+      pushModeLabel: "Read only",
+      pushModeDescription:
+        "Notion database sources are read-only in this pilot.",
+      notes:
+        "Notion values are refreshed through the current user's OAuth connection; write-back is disabled.",
+      label: sourceMetadataLabel(args.sourceType, args.sourceTable),
+    });
+  }
   return JSON.stringify({
     primaryKey: "id",
     titleField: "title",
@@ -3180,6 +3206,21 @@ export function sourceCapabilitiesForType(
       readOnlyRefresh: true,
     });
   }
+  if (sourceType === "notion-database") {
+    return serializeSourceCapabilitiesRecord({
+      canRefresh: true,
+      canCreateChangeSets: false,
+      canWriteFields: false,
+      canWriteBody: false,
+      canPush: false,
+      canPull: true,
+      canPublish: false,
+      canDelete: false,
+      canStageLocalRevision: false,
+      liveWritesEnabled: false,
+      readOnlyRefresh: true,
+    });
+  }
   return serializeSourceCapabilitiesRecord();
 }
 
@@ -4168,82 +4209,106 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
   const existingTitles = new Set(
     currentItems.map((row) => row.document.title.trim().toLowerCase()),
   );
-  const [maxDocPos] = await db
-    .select({ max: sql<number>`COALESCE(MAX(position), -1)` })
-    .from(schema.documents)
-    .where(
-      and(
-        eq(schema.documents.ownerEmail, args.database.ownerEmail),
-        eq(schema.documents.parentId, args.database.documentId),
-      ),
-    );
-  const [maxItemPos] = await db
-    .select({ max: sql<number>`COALESCE(MAX(position), -1)` })
-    .from(schema.contentDatabaseItems)
-    .where(eq(schema.contentDatabaseItems.databaseId, args.database.id));
-
-  let nextDocPosition = (maxDocPos?.max ?? -1) + 1;
-  let nextItemPosition = (maxItemPos?.max ?? -1) + 1;
-  const documentRows: (typeof schema.documents.$inferInsert)[] = [];
-  const itemRows: (typeof schema.contentDatabaseItems.$inferInsert)[] = [];
-
-  for (const entry of args.entries) {
-    if (
-      builderCmsEntryAlreadyRepresented({
-        entry,
-        sourceTable: args.sourceTable,
-        existingSourceRows: args.existingSourceRows ?? [],
-      })
-    ) {
-      continue;
-    }
 
-    const title = entry.title.trim() || entry.id;
-    const titleKey = title.toLowerCase();
-    if (!args.skipTitleDedup && existingTitles.has(titleKey)) continue;
-    existingTitles.add(titleKey);
-
-    const documentId = nanoid();
-    const itemId = nanoid();
-    documentRows.push({
-      id: documentId,
-      ownerEmail: args.database.ownerEmail,
-      orgId: args.database.orgId,
-      parentId: args.database.documentId,
-      title,
-      content: "",
-      icon: null,
-      position: nextDocPosition++,
-      isFavorite: 0,
-      hideFromSearch: databaseDocument?.hideFromSearch ?? 0,
-      visibility: databaseDocument?.visibility ?? "private",
-      createdAt: args.now,
-      updatedAt: args.now,
-    });
-    itemRows.push({
-      id: itemId,
-      ownerEmail: args.database.ownerEmail,
-      orgId: args.database.orgId,
-      databaseId: args.database.id,
-      documentId,
-      position: nextItemPosition++,
-      bodyHydrationStatus: "pending",
-      bodyHydrationError: null,
-      createdAt: args.now,
-      updatedAt: args.now,
-    });
-    importedEntriesByDocumentId.set(documentId, entry);
-  }
-  await db.transaction(async (tx) => {
-    for (const chunk of chunks(documentRows, bulkChunkSizeForColumnCount(13))) {
-      await tx.insert(schema.documents).values(chunk);
-    }
-    for (const chunk of chunks(itemRows, bulkChunkSizeForColumnCount(10))) {
-      await tx.insert(schema.contentDatabaseItems).values(chunk);
-    }
-  });
+  // Reads MAX(position) for both `documents` and `content_database_items`
+  // then batch-inserts at MAX+1.. — serialize the whole read-through-write
+  // span per scope so a concurrent import/add/move targeting the same parent
+  // document or the same database can't read the same MAX (see
+  // _position-utils.ts).
+  return withPositionLock(
+    documentsPositionScope(args.database.ownerEmail, args.database.documentId),
+    () =>
+      withPositionLock(
+        databaseItemsPositionScope(args.database.id),
+        async () => {
+          const [maxDocPos] = await db
+            .select({ max: sql<number>`COALESCE(MAX(position), -1)` })
+            .from(schema.documents)
+            .where(
+              and(
+                eq(schema.documents.ownerEmail, args.database.ownerEmail),
+                eq(schema.documents.parentId, args.database.documentId),
+              ),
+            );
+          const [maxItemPos] = await db
+            .select({ max: sql<number>`COALESCE(MAX(position), -1)` })
+            .from(schema.contentDatabaseItems)
+            .where(
+              eq(schema.contentDatabaseItems.databaseId, args.database.id),
+            );
 
-  return { imported: itemRows.length, importedEntriesByDocumentId };
+          let nextDocPosition = (maxDocPos?.max ?? -1) + 1;
+          let nextItemPosition = (maxItemPos?.max ?? -1) + 1;
+          const documentRows: (typeof schema.documents.$inferInsert)[] = [];
+          const itemRows: (typeof schema.contentDatabaseItems.$inferInsert)[] =
+            [];
+
+          for (const entry of args.entries) {
+            if (
+              builderCmsEntryAlreadyRepresented({
+                entry,
+                sourceTable: args.sourceTable,
+                existingSourceRows: args.existingSourceRows ?? [],
+              })
+            ) {
+              continue;
+            }
+
+            const title = entry.title.trim() || entry.id;
+            const titleKey = title.toLowerCase();
+            if (!args.skipTitleDedup && existingTitles.has(titleKey)) continue;
+            existingTitles.add(titleKey);
+
+            const documentId = nanoid();
+            const itemId = nanoid();
+            documentRows.push({
+              id: documentId,
+              ownerEmail: args.database.ownerEmail,
+              orgId: args.database.orgId,
+              parentId: args.database.documentId,
+              title,
+              content: "",
+              icon: null,
+              position: nextDocPosition++,
+              isFavorite: 0,
+              hideFromSearch: databaseDocument?.hideFromSearch ?? 0,
+              visibility: databaseDocument?.visibility ?? "private",
+              createdAt: args.now,
+              updatedAt: args.now,
+            });
+            itemRows.push({
+              id: itemId,
+              ownerEmail: args.database.ownerEmail,
+              orgId: args.database.orgId,
+              databaseId: args.database.id,
+              documentId,
+              position: nextItemPosition++,
+              bodyHydrationStatus: "pending",
+              bodyHydrationError: null,
+              createdAt: args.now,
+              updatedAt: args.now,
+            });
+            importedEntriesByDocumentId.set(documentId, entry);
+          }
+          await db.transaction(async (tx) => {
+            for (const chunk of chunks(
+              documentRows,
+              bulkChunkSizeForColumnCount(13),
+            )) {
+              await tx.insert(schema.documents).values(chunk);
+            }
+            for (const chunk of chunks(
+              itemRows,
+              bulkChunkSizeForColumnCount(10),
+            )) {
+              await tx.insert(schema.contentDatabaseItems).values(chunk);
+            }
+          });
+
+          return { imported: itemRows.length, importedEntriesByDocumentId };
+        },
+      ),
+  );
 }
 
 export async function resyncBuilderCmsSourceSnapshot(args: {
@@ -4825,7 +4890,10 @@ export async function storeSecondarySourceRows(args: {
       sourceDisplayKey:
         entry.title?.trim() || `${args.sourceTable}-${index + 1}`,
       sourceValuesJson: JSON.stringify(entry.sourceValues ?? {}),
-      provenance: "secondary source row",
+      provenance:
+        args.sourceType === "notion-database"
+          ? "Notion database read adapter"
+          : "secondary source row",
       syncState: "linked" as const,
       freshness: "fresh" as const,
       lastSyncedAt: args.now,
@@ -4844,6 +4912,7 @@ export async function storeSecondarySourceRows(args: {
 export async function seedSecondarySourceFields(args: {
   sourceId: string;
   ownerEmail: string;
+  sourceType?: ContentDatabaseSourceType;
   modelFields: BuilderCmsModelFieldSummary[];
   sampleEntry?: BuilderCmsSourceEntry;
   now: string;
@@ -4909,7 +4978,10 @@ export async function seedSecondarySourceFields(args: {
         mappingType: "property" as const,
         writeOwner: "source" as const,
         readOnly: 1,
-        provenance: "secondary source field",
+        provenance:
+          args.sourceType === "notion-database"
+            ? "Notion database read adapter"
+            : "secondary source field",
         freshness: "fresh" as const,
         lastSyncedAt: args.now,
         createdAt: existing?.createdAt ?? args.now,
@@ -5008,6 +5080,39 @@ export async function updateBuilderCmsSourceReadMetadata(args: {
     .where(eq(schema.contentDatabaseSources.id, args.sourceId));
 }
 
+export async function updateReadOnlySourceMetadata(args: {
+  sourceId: string;
+  sourceType: ContentDatabaseSourceType;
+  sourceTable: string;
+  fetchedAt: string;
+  now: string;
+  message: string | null;
+  metadata?: Record<string, unknown>;
+  syncState?: ContentDatabaseSourceSyncState;
+}) {
+  const db = getDb();
+  const [current] = await db
+    .select({ metadataJson: schema.contentDatabaseSources.metadataJson })
+    .from(schema.contentDatabaseSources)
+    .where(eq(schema.contentDatabaseSources.id, args.sourceId))
+    .limit(1);
+  const existing =
+    parseObject<Record<string, unknown>>(current?.metadataJson) ?? {};
+  await db
+    .update(schema.contentDatabaseSources)
+    .set({
+      syncState: args.syncState ?? (args.message ? "refreshing" : "linked"),
+      freshness: args.message ? "stale" : "fresh",
+      capabilitiesJson: sourceCapabilitiesForType(args.sourceType),
+      metadataJson: JSON.stringify({ ...existing, ...(args.metadata ?? {}) }),
+      lastRefreshedAt: args.now,
+      lastSourceUpdatedAt: args.fetchedAt,
+      lastError: null,
+      updatedAt: args.now,
+    })
+    .where(eq(schema.contentDatabaseSources.id, args.sourceId));
+}
+
 export async function getExistingSource(databaseId: string) {
   const db = getDb();
   const [source] = await db
@@ -5154,26 +5259,31 @@ export async function ensureDatabaseSourceProperty(args: {
       .set({ optionsJson, updatedAt: args.now })
       .where(eq(schema.documentPropertyDefinitions.id, existing.id));
   } else {
-    const [maxPos] = await db
-      .select({ max: sql<number>`COALESCE(MAX(position), -1)` })
-      .from(schema.documentPropertyDefinitions)
-      .where(
-        eq(schema.documentPropertyDefinitions.databaseId, args.database.id),
-      );
     propertyId = crypto.randomUUID();
-    await db.insert(schema.documentPropertyDefinitions).values({
-      id: propertyId,
-      ownerEmail: args.database.ownerEmail,
-      orgId: args.database.orgId,
-      databaseId: args.database.id,
-      name: SOURCE_PROPERTY_NAME,
-      type: "select",
-      visibility: "always_show",
-      optionsJson,
-      position: (maxPos?.max ?? -1) + 1,
-      createdAt: args.now,
-      updatedAt: args.now,
-    });
+    await withPositionLock(
+      propertyDefinitionsPositionScope(args.database.id),
+      async () => {
+        const [maxPos] = await db
+          .select({ max: sql<number>`COALESCE(MAX(position), -1)` })
+          .from(schema.documentPropertyDefinitions)
+          .where(
+            eq(schema.documentPropertyDefinitions.databaseId, args.database.id),
+          );
+        await db.insert(schema.documentPropertyDefinitions).values({
+          id: propertyId,
+          ownerEmail: args.database.ownerEmail,
+          orgId: args.database.orgId,
+          databaseId: args.database.id,
+          name: SOURCE_PROPERTY_NAME,
+          type: "select",
+          visibility: "always_show",
+          optionsJson,
+          position: (maxPos?.max ?? -1) + 1,
+          createdAt: args.now,
+          updatedAt: args.now,
+        });
+      },
+    );
   }
 
   // A row's Source value IS its owning source id (= the option id); unsourced

package/corpus/templates/content/actions/_notion-database-source-adapter.ts

@@ -0,0 +1,216 @@
+import { getRequestUserEmail } from "@agent-native/core/server";
+
+import {
+  getNotionConnectionForOwner,
+  notionFetch,
+} from "../server/lib/notion.js";
+import type {
+  BuilderCmsModelFieldSummary,
+  DocumentPropertyValue,
+} from "../shared/api.js";
+import type { BuilderCmsSourceEntry } from "./_builder-cms-source-adapter.js";
+import type { ContentDatabaseSourceReadResult } from "./_content-database-source-adapters.js";
+
+const NOTION_SOURCE_PAGE_SIZE = 100;
+const NOTION_SOURCE_MAX_PAGES = 5;
+
+type NotionProperty = {
+  id?: string;
+  name?: string;
+  type?: string;
+  [key: string]: unknown;
+};
+
+type NotionPage = {
+  object?: string;
+  id: string;
+  url?: string;
+  last_edited_time?: string;
+  properties?: Record<string, NotionProperty>;
+};
+
+function plainText(value: unknown): string {
+  return Array.isArray(value)
+    ? value
+        .map((part) =>
+          typeof part === "object" && part && "plain_text" in part
+            ? String((part as { plain_text?: unknown }).plain_text ?? "")
+            : "",
+        )
+        .join("")
+    : "";
+}
+
+const supportedTypes = new Set([
+  "title",
+  "rich_text",
+  "number",
+  "checkbox",
+  "url",
+  "email",
+  "phone_number",
+  "date",
+  "select",
+  "status",
+  "multi_select",
+]);
+
+function sourceFieldType(type: string) {
+  if (!supportedTypes.has(type)) return `unsupported:${type || "unknown"}`;
+  if (type === "rich_text" || type === "title") return "text";
+  if (type === "phone_number") return "phone";
+  return type;
+}
+
+function propertyValue(property: NotionProperty): DocumentPropertyValue {
+  const type = property.type ?? "unknown";
+  if (!supportedTypes.has(type)) {
+    return `[Unsupported Notion property: ${type}]`;
+  }
+  if (type === "title" || type === "rich_text")
+    return plainText(property[type]);
+  if (type === "number")
+    return typeof property.number === "number" ? property.number : null;
+  if (type === "checkbox") return property.checkbox === true;
+  if (type === "url" || type === "email" || type === "phone_number") {
+    const value = property[type];
+    return typeof value === "string" ? value : null;
+  }
+  if (type === "date") {
+    const date = property.date as
+      | { start?: unknown; end?: unknown }
+      | null
+      | undefined;
+    return typeof date?.start === "string"
+      ? {
+          start: date.start,
+          end: typeof date.end === "string" ? date.end : null,
+          includeTime: date.start.includes("T"),
+        }
+      : null;
+  }
+  if (type === "select" || type === "status") {
+    const option = property[type] as { name?: unknown } | null | undefined;
+    return typeof option?.name === "string" ? option.name : null;
+  }
+  if (type === "multi_select") {
+    return Array.isArray(property.multi_select)
+      ? property.multi_select.flatMap((option) =>
+          typeof option === "object" &&
+          option &&
+          "name" in option &&
+          typeof (option as { name?: unknown }).name === "string"
+            ? [(option as { name: string }).name]
+            : [],
+        )
+      : [];
+  }
+  return null;
+}
+
+function titleFromProperties(properties: Record<string, NotionProperty>) {
+  const title = Object.values(properties).find(
+    (property) => property.type === "title",
+  );
+  return title ? plainText(title.title) || "Untitled" : "Untitled";
+}
+
+export async function readNotionDatabaseSource(args: {
+  sourceTable: string;
+  limit: number;
+  offset: number;
+  fullRefresh?: boolean;
+}): Promise<ContentDatabaseSourceReadResult> {
+  if (args.offset !== 0) {
+    throw new Error(
+      "Notion database sources use cursor pagination; numeric offsets are not supported.",
+    );
+  }
+  const userEmail = getRequestUserEmail();
+  if (!userEmail)
+    throw new Error("Notion database access requires a signed-in user.");
+  const connection = await getNotionConnectionForOwner(userEmail);
+  if (!connection)
+    throw new Error("Connect Notion before attaching a Notion database.");
+
+  const dataSource = await notionFetch<{
+    id: string;
+    title?: Array<{ plain_text?: string }>;
+    properties?: Record<string, NotionProperty>;
+  }>(
+    `/data_sources/${encodeURIComponent(args.sourceTable)}`,
+    connection.accessToken,
+  );
+  const fields: BuilderCmsModelFieldSummary[] = Object.entries(
+    dataSource.properties ?? {},
+  ).map(([name, property]) => ({
+    name: property.id || name,
+    label: property.name || name,
+    type: sourceFieldType(property.type ?? "unknown"),
+    required: false,
+  }));
+
+  const entries: BuilderCmsSourceEntry[] = [];
+  let cursor: string | undefined;
+  let hasMore = false;
+  const maxPages = args.fullRefresh ? NOTION_SOURCE_MAX_PAGES : 1;
+  for (let page = 0; page < maxPages; page++) {
+    const response = await notionFetch<{
+      results?: NotionPage[];
+      has_more?: boolean;
+      next_cursor?: string | null;
+    }>(
+      `/data_sources/${encodeURIComponent(args.sourceTable)}/query`,
+      connection.accessToken,
+      {
+        method: "POST",
+        body: JSON.stringify({
+          page_size: Math.min(NOTION_SOURCE_PAGE_SIZE, args.limit),
+          ...(cursor ? { start_cursor: cursor } : {}),
+        }),
+      },
+    );
+    for (const result of response.results ?? []) {
+      if (result.object && result.object !== "page") continue;
+      const properties = result.properties ?? {};
+      entries.push({
+        id: result.id,
+        model: args.sourceTable,
+        title: titleFromProperties(properties),
+        urlPath: result.url ?? "",
+        updatedAt: result.last_edited_time ?? "",
+        sourceValues: Object.fromEntries(
+          Object.entries(properties).map(([name, property]) => [
+            property.id || name,
+            propertyValue(property),
+          ]),
+        ),
+      });
+    }
+    hasMore = response.has_more === true && !!response.next_cursor;
+    cursor = response.next_cursor ?? undefined;
+    if (!hasMore || entries.length >= args.limit) break;
+  }
+
+  const fetchedAt = new Date().toISOString();
+  return {
+    state: "live",
+    entries: entries.slice(0, args.limit),
+    fields,
+    fetchedAt,
+    message: hasMore
+      ? `Loaded a bounded Notion snapshot; more rows remain after cursor ${cursor ? "present" : "missing"}.`
+      : null,
+    metadata: {
+      provider: "notion",
+      dataSourceId: dataSource.id,
+      dataSourceName: plainText(dataSource.title) || "Notion database",
+      hasMore,
+      nextCursor: cursor ?? null,
+      fetchedRowCount: Math.min(entries.length, args.limit),
+      unsupportedPropertyCount: fields.filter((field) =>
+        field.type.startsWith("unsupported:"),
+      ).length,
+    },
+  };
+}

package/corpus/templates/content/actions/_position-utils.ts

@@ -0,0 +1,76 @@
+// Serializes "read MAX(position) among siblings, then insert/update at
+// MAX+1" sequences so two concurrent writers (agent + human, two browser
+// tabs, an import job racing a manual add) never read the same MAX and
+// persist duplicate `position` values. `position` columns in this app have
+// no unique/sequence constraint (see templates/content/server/db/schema.ts),
+// so ordering is only as stable as the read-then-insert stays atomic.
+//
+// Mirrors the per-deck lock in templates/slides/actions/patch-deck.ts: an
+// in-process, globalThis-keyed promise chain per scope key. This only
+// serializes writers within THIS process, but that is sufficient here
+// because every write path already goes through the app's own action/route
+// layer in the same server process — it is not a substitute for a DB-level
+// unique constraint, but it closes the read-then-write race that produces
+// duplicate positions today.
+//
+// Sites that already compute MAX(position) and insert inside the SAME
+// `db.transaction()` (e.g. submit-content-database-form.ts,
+// add-content-database-source-field-property.ts) do not need this helper —
+// wrap new sites in the lock instead of introducing a second mechanism.
+
+const LOCK_KEY = "__contentPositionLocks" as const;
+type GlobalWithLocks = typeof globalThis & {
+  [LOCK_KEY]?: Map<string, Promise<unknown>>;
+};
+const globalRef = globalThis as GlobalWithLocks;
+if (!globalRef[LOCK_KEY]) {
+  globalRef[LOCK_KEY] = new Map<string, Promise<unknown>>();
+}
+const positionLocks: Map<string, Promise<unknown>> = globalRef[LOCK_KEY]!;
+
+/** Run `fn` serialized against any other call using the same `scopeKey`. */
+export function withPositionLock<T>(
+  scopeKey: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const prev = positionLocks.get(scopeKey) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  positionLocks.set(scopeKey, next);
+  next
+    .finally(() => {
+      if (positionLocks.get(scopeKey) === next) positionLocks.delete(scopeKey);
+    })
+    .catch(() => {});
+  return next;
+}
+
+/**
+ * Lock scope for `documents.position` siblings under one owner + parent
+ * (or one owner's root level when `parentId` is null/undefined). Every call
+ * site that computes MAX(position) over `documents` scoped by
+ * (ownerEmail, parentId) should share this key so concurrent inserts under
+ * the same parent — regardless of which action/route triggers them — are
+ * serialized against each other.
+ */
+export function documentsPositionScope(
+  ownerEmail: string,
+  parentId: string | null | undefined,
+): string {
+  return `documents:${ownerEmail}:${parentId ?? "root"}`;
+}
+
+/**
+ * Lock scope for `content_database_items.position` rows within one
+ * database.
+ */
+export function databaseItemsPositionScope(databaseId: string): string {
+  return `contentDatabaseItems:${databaseId}`;
+}
+
+/**
+ * Lock scope for `document_property_definitions.position` rows within one
+ * database.
+ */
+export function propertyDefinitionsPositionScope(databaseId: string): string {
+  return `documentPropertyDefinitions:${databaseId}`;
+}