@agent-native/core 0.85.7 → 0.86.1

package/corpus/templates/clips/server/lib/recording-upload-state.ts

@@ -22,6 +22,62 @@ function numberFromRowValue(value: unknown): number {
   return 0;
 }
 
+export interface RecordingChunkKey {
+  key: string;
+  index: number;
+}
+
+export function recordingChunkIndexFromKey(key: string): number | null {
+  const rawIndex = key.slice(key.lastIndexOf("-") + 1);
+  if (!/^\d+$/.test(rawIndex)) return null;
+  const index = Number(rawIndex);
+  return Number.isSafeInteger(index) ? index : null;
+}
+
+export function validateRecordingChunkKeys(
+  keys: string[],
+  expectedChunks?: number,
+): RecordingChunkKey[] {
+  const parsed = keys.map((key) => {
+    const index = recordingChunkIndexFromKey(key);
+    if (index === null) {
+      throw new Error(
+        `Recording upload contains an invalid chunk key (${key}). Please retry the recording.`,
+      );
+    }
+    return { key, index };
+  });
+
+  parsed.sort((a, b) => a.index - b.index);
+
+  for (let i = 0; i < parsed.length; i++) {
+    const chunk = parsed[i]!;
+    if (chunk.index < i) {
+      throw new Error(
+        `Recording upload contains duplicate chunk ${chunk.index}. Please retry the recording.`,
+      );
+    }
+    if (chunk.index > i) {
+      throw new Error(
+        `Recording upload is incomplete: missing chunk ${i}. Please retry the recording.`,
+      );
+    }
+  }
+
+  if (
+    typeof expectedChunks === "number" &&
+    Number.isSafeInteger(expectedChunks) &&
+    expectedChunks >= 0 &&
+    parsed.length !== expectedChunks
+  ) {
+    throw new Error(
+      `Recording upload is incomplete (${parsed.length} of ${expectedChunks} chunks received). Please retry the recording.`,
+    );
+  }
+
+  return parsed;
+}
+
 export async function listRecordingChunkKeys(
   ownerEmail: string,
   recordingId: string,

package/corpus/templates/clips/server/plugins/db.ts

@@ -1,4 +1,9 @@
-import { runMigrations, getDbExec, isPostgres } from "@agent-native/core/db";
+import {
+  runMigrations,
+  getDbExec,
+  isPostgres,
+  ensureAdditiveColumns,
+} from "@agent-native/core/db";
 import { registerEvent } from "@agent-native/core/event-bus";
 import { z } from "zod";
 
@@ -8,6 +13,25 @@ import { z } from "zod";
 // are loaded in a separate Vite SSR bundle from user actions, so we trigger
 // the registration eagerly from the always-loaded db plugin.
 import "../db/index.js";
+import * as schema from "../db/schema.js";
+
+/**
+ * Every Drizzle table exported from schema.ts. Filters out type-only and
+ * helper exports the same way db.spec.ts's `isDrizzleTable` regression guard
+ * does: a real table carries a Symbol-keyed drizzle metadata bag, plain
+ * exports don't.
+ */
+function isDrizzleTable(value: unknown): value is object {
+  return (
+    !!value &&
+    typeof value === "object" &&
+    Object.getOwnPropertySymbols(value).some((s) =>
+      s.toString().includes("drizzle"),
+    )
+  );
+}
+
+const schemaTables = Object.values(schema).filter(isDrizzleTable);
 
 /**
  * Post-migration fixup for Postgres: retype boolean-mode columns from bigint
@@ -63,6 +87,10 @@ async function retypeBooleanColumnsOnPostgres(): Promise<void> {
   }
 }
 
+// Convention: every new migration below MUST set a unique `name:` slug (see
+// packages/core/src/db/migrations.ts for the full rationale). Version numbers
+// alone are not a safe identity across parallel branches that each extend
+// this list independently — see the v41 incident documented on v41 below.
 const migrations = runMigrations(
   [
     // ---------------------------------------------------------------------------
@@ -648,9 +676,19 @@ const migrations = runMigrations(
     // and `dictation_shares` (v25) are NOT on any access path — the schema and
     // every `accessFilter` callsite use the `clips_*` prefixed tables — so they
     // are intentionally skipped.
+    //
+    // v41 was recorded as applied in `clips_migrations` on the shared Neon
+    // database, but none of its 8 indexes actually existed live (confirmed via
+    // `pg_indexes` — the exact "recorded but never ran" collision class
+    // `runMigrations` name-based tracking exists to fix; see
+    // packages/core/src/db/migrations.ts). All statements here are
+    // `CREATE INDEX IF NOT EXISTS` (unchanged, still idempotent), so it is
+    // named to re-apply by name regardless of this database's recorded
+    // MAX(version).
     // -------------------------------------------------------------------------
     {
       version: 41,
+      name: "recordings-comments-shares-hot-path-indexes",
       sql: [
         // recordings list: library view filters owner_email + workspace_id and
         // sorts by created_at; the accessFilter owner branch also scopes by
@@ -1402,6 +1440,15 @@ async function backfillLegacyClipsTables(): Promise<void> {
   }
 }
 
+/**
+ * The migration list above is the authoritative source for tables, indexes,
+ * and data transforms. `ensureAdditiveColumns` runs after it (and after the
+ * other startup backfills) as a belt-and-braces safety net: a column added to
+ * schema.ts without a matching hand-written ALTER migration silently 500s
+ * every query touching a pre-existing production table. It only ever adds
+ * missing columns — never drops, renames, or retypes anything — and any
+ * failure here is logged and swallowed so it can never fail boot.
+ */
 export default async (nitroApp: any): Promise<void> => {
   await migrations(nitroApp);
   await retypeBooleanColumnsOnPostgres();
@@ -1413,6 +1460,26 @@ export default async (nitroApp: any): Promise<void> => {
     console.warn("[db] chunk sweep failed:", (err as Error)?.message ?? err);
   });
 
+  try {
+    const summary = await ensureAdditiveColumns({
+      db: getDbExec(),
+      tables: schemaTables,
+    });
+    if (summary.errors.length > 0) {
+      console.warn(
+        "[db] ensureAdditiveColumns completed with errors:",
+        summary.errors,
+      );
+    }
+  } catch (err) {
+    // Never fail boot over the safety net itself — the authoritative
+    // migrations above already ran.
+    console.warn(
+      "[db] ensureAdditiveColumns failed (non-fatal):",
+      err instanceof Error ? err.message : err,
+    );
+  }
+
   // ---------------------------------------------------------------------------
   // Register Clips template events for the automations system.
   // ---------------------------------------------------------------------------

package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/chunk.post.ts

@@ -96,6 +96,13 @@ function stateNumber(
   return typeof raw === "number" && Number.isFinite(raw) ? raw : undefined;
 }
 
+function expectedDataChunksForFinalPost(
+  index: number,
+  bodySize: number,
+): number {
+  return index + (bodySize > 0 ? 1 : 0);
+}
+
 function trackUploadBlockingFailure(
   ownerEmail: string,
   properties: Record<string, unknown>,
@@ -145,7 +152,7 @@ export default defineEventHandler(async (event: H3Event) => {
     mimeType,
   });
 
-  if (!Number.isFinite(index) || index < 0) {
+  if (!Number.isFinite(index) || !Number.isInteger(index) || index < 0) {
     throw createError({ statusCode: 400, message: "Invalid chunk index" });
   }
 
@@ -286,6 +293,9 @@ export default defineEventHandler(async (event: H3Event) => {
     // readRawBody(event, false) returns Uint8Array. Buffer is a Uint8Array
     // subclass on Node, so this is safe whether we're on Node or workerd.
     const bytes: Uint8Array = raw ?? new Uint8Array(0);
+    const expectedDataChunks = isFinal
+      ? expectedDataChunksForFinalPost(index, bytes.byteLength)
+      : undefined;
 
     const uploadStateRaw = await readAppState(
       `recording-upload-${recordingId}`,
@@ -409,6 +419,13 @@ export default defineEventHandler(async (event: H3Event) => {
         progress,
         chunksReceived: index + 1,
         totalChunks: total,
+        ...(expectedDataChunks !== undefined
+          ? {
+              expectedDataChunks,
+              finalChunkIndex: index,
+              finalChunkBytes: bytes.byteLength,
+            }
+          : {}),
         bytesReceived,
         maxBytes: MAX_RECORDING_UPLOAD_BYTES,
         mimeType,
@@ -426,13 +443,20 @@ export default defineEventHandler(async (event: H3Event) => {
             eq(schema.recordings.status, existing.status),
           ),
         );
-    } else if (bytes.byteLength > 0) {
+    } else if (bytes.byteLength > 0 || isFinal) {
       const failedResponse = await stopIfUploadFailed();
       if (failedResponse) return failedResponse;
       await writeAppState(`recording-upload-${recordingId}`, {
         recordingId,
-        status: "uploading",
+        status: isFinal ? "processing" : "uploading",
         chunksReceived: index + 1,
+        ...(expectedDataChunks !== undefined
+          ? {
+              expectedDataChunks,
+              finalChunkIndex: index,
+              finalChunkBytes: bytes.byteLength,
+            }
+          : {}),
         bytesReceived,
         maxBytes: MAX_RECORDING_UPLOAD_BYTES,
         mimeType,

package/corpus/templates/content/.agents/skills/real-time-sync/SKILL.md

@@ -58,6 +58,21 @@ The agent modifies data in SQL, but the UI runs in the browser. SSE bridges same
 
 - Don't create manual polling loops — `useDbSync()` handles SSE plus fallback polling
 - Don't create your own fetch-based polling alongside `useDbSync` — use the `onEvent` callback for custom handling
+- Don't open your own `EventSource` to `/_agent-native/events`. A tab must hold exactly ONE SSE connection no matter how many features listen — extra streams eat the browser's per-origin connection budget and can starve ordinary data fetches (worst on HTTP/1.1 dev servers). Subscribe to the shared transport instead:
+
+  ```ts
+  import { subscribeSyncEvents } from "@agent-native/core/client";
+
+  const unsubscribe = subscribeSyncEvents({
+    onEvents: (events) => {
+      // filter by event.source and handle push-style updates
+    },
+    // Optional: relax your own fallback cadence while push is healthy.
+    onSseStateChange: (connected) => {},
+  });
+  ```
+
+  `useDbSync` and every `subscribeSyncEvents` subscriber share one `EventSource` and one fallback poll loop per tab — this is how collaborative documents receive doc updates and cursor/awareness events.
 
 ## Which sources to depend on
 

package/corpus/templates/content/.agents/skills/storing-data/SKILL.md

@@ -25,6 +25,14 @@ Agent-native apps use Drizzle ORM over the configured SQL backend. Local develop
 
 For app code, use Drizzle's schema/query DSL by default. Raw SQL is an escape hatch for additive migrations, health checks, or one-off maintenance, not the normal way to build features.
 
+### Naming migrations
+
+When you add an entry to a `runMigrations([...])` list (`@agent-native/core/db`), always give it a unique `name:` slug (e.g. `name: "analytics-alert-rules-table"`) alongside its `version`. Never renumber or reuse version numbers on existing entries.
+
+Why: version numbers alone are not a safe identity. Two branches that each independently extend the same migration list can ship different DDL under the same version numbers — whichever branch deploys first "claims" those version numbers in the bookkeeping table, and the other branch's DDL is silently treated as already applied even though it never ran. This exact collision took down analytics: parallel branches both extended their migration list through v75-v83 with different DDL, so `analytics_alert_rules`, `analytics_alert_incidents`, and `session_recordings.network_error_count` never made it to production despite the bookkeeping table showing every version as applied. A `name:` slug is tracked independently of version numbers, so it applies exactly once per database regardless of what any other branch already recorded.
+
+Existing unnamed migrations don't need to be renamed retroactively (the two gating strategies coexist), but any new entry should always carry a name.
+
 ### Core SQL Stores (auto-created, available in all templates)
 
 | Store               | Purpose                                              | Access                                     |

package/corpus/templates/content/actions/edit-document.ts

@@ -1,5 +1,10 @@
 import { defineAction } from "@agent-native/core";
 import { writeAppState } from "@agent-native/core/application-state";
+import {
+  agentTouchDocument,
+  hasCollabState,
+  searchAndReplace,
+} from "@agent-native/core/collab";
 import { assertAccess } from "@agent-native/core/sharing";
 import { eq } from "drizzle-orm";
 import { z } from "zod";
@@ -32,10 +37,16 @@ export default defineAction({
       .describe("JSON array of {find, replace} objects (batch mode)"),
   }),
   http: false,
-  run: async (args) => {
+  run: async (args, ctx) => {
     const id = args.id;
     if (!id) throw new Error("--id is required");
 
+    // Only publish AI presence for genuine agent invocations (in-app tool loop,
+    // sub-agents/A2A → "tool"; external MCP agents → "mcp"). A browser or
+    // programmatic call must never light the "AI editing" flag.
+    const isAgentCaller =
+      ctx?.caller === "tool" || ctx?.caller === "mcp" || ctx?.caller === "a2a";
+
     let edits: TextEdit[];
 
     if (args.edits) {
@@ -91,6 +102,7 @@ export default defineAction({
     let content: string = existing.content ?? "";
     const results: string[] = [];
     let changeCount = 0;
+    const appliedEdits: TextEdit[] = [];
 
     for (const edit of edits) {
       const idx = content.indexOf(edit.find);
@@ -105,6 +117,7 @@ export default defineAction({
         edit.replace +
         content.slice(idx + edit.find.length);
       changeCount++;
+      appliedEdits.push(edit);
       const action = edit.replace === "" ? "deleted" : "replaced";
       results.push(
         `${action}: "${edit.find.slice(0, 40)}${edit.find.length > 40 ? "..." : ""}"`,
@@ -133,6 +146,40 @@ export default defineAction({
       .set({ content, updatedAt: new Date().toISOString() })
       .where(eq(schema.documents.id, id));
 
+    // Make the agent edit VISIBLE as a live collaborator. Content's collab doc
+    // binds the TipTap Y.XmlFragment("default"), so a live editing session is
+    // patched surgically through `searchAndReplace` (which also auto-publishes
+    // agent presence + a lingering recent-edit highlight on the changed text).
+    // When no collab session exists (no editor open, XmlFragment unseeded) the
+    // SQL write above is authoritative and reconciles on next open; we still
+    // touch agent presence best-effort so a viewer who opens the doc mid-linger
+    // sees the AI flag. All of this is wrapped so presence never fails the edit.
+    if (isAgentCaller) {
+      try {
+        if (await hasCollabState(id)) {
+          for (const edit of appliedEdits) {
+            await searchAndReplace(id, edit.find, edit.replace, "agent");
+          }
+        } else {
+          const firstChange = appliedEdits.find((e) => e.replace)?.replace;
+          agentTouchDocument(id, {
+            edit: {
+              descriptor: {
+                kind: "text",
+                quote: (firstChange ?? appliedEdits[0]?.find ?? "").slice(
+                  0,
+                  80,
+                ),
+              },
+              label: existing.title || undefined,
+            },
+          });
+        }
+      } catch (error) {
+        console.error("edit-document: agent presence publish failed", error);
+      }
+    }
+
     await writeAppState("refresh-signal", { ts: Date.now() });
 
     return {

package/corpus/templates/content/actions/list-documents.ts

@@ -98,22 +98,21 @@ export default defineAction({
 
     const shareRoleByDocumentId = new Map<string, ShareRole>();
     const notionPageIdByDocumentId = new Map<string, string>();
-    if (documents.length > 0) {
-      const notionLinks = await db
-        .select({
-          documentId: schema.documentSyncLinks.documentId,
-          remotePageId: schema.documentSyncLinks.remotePageId,
-        })
-        .from(schema.documentSyncLinks)
-        .where(
-          inArray(
-            schema.documentSyncLinks.documentId,
-            documents.map((d) => d.id),
-          ),
-        );
-      for (const link of notionLinks) {
-        notionPageIdByDocumentId.set(link.documentId, link.remotePageId);
+    const databaseByDocumentId = new Map<
+      string,
+      typeof schema.contentDatabases.$inferSelect
+    >();
+    const databaseMembershipByDocumentId = new Map<
+      string,
+      {
+        item: typeof schema.contentDatabaseItems.$inferSelect;
+        database: typeof schema.contentDatabases.$inferSelect;
       }
+    >();
+    const softDeletedDocumentIds = new Set<string>();
+
+    if (documents.length > 0) {
+      const visibleDocumentIds = documents.map((d) => d.id);
 
       const principalClauses: NonNullable<ReturnType<typeof and>>[] = [];
       if (userEmail) {
@@ -133,97 +132,105 @@ export default defineAction({
         );
       }
 
-      if (principalClauses.length > 0) {
-        const shareRows = await db
+      // These queries all depend only on the initial `documents` id list
+      // (already fetched above), not on each other's results, so they run
+      // concurrently instead of as sequential round-trips.
+      const [
+        notionLinks,
+        shareRows,
+        databases,
+        databaseMemberships,
+        softDeletedDatabases,
+      ] = await Promise.all([
+        db
+          .select({
+            documentId: schema.documentSyncLinks.documentId,
+            remotePageId: schema.documentSyncLinks.remotePageId,
+          })
+          .from(schema.documentSyncLinks)
+          .where(
+            inArray(schema.documentSyncLinks.documentId, visibleDocumentIds),
+          ),
+        principalClauses.length > 0
+          ? db
+              .select({
+                resourceId: schema.documentShares.resourceId,
+                role: schema.documentShares.role,
+              })
+              .from(schema.documentShares)
+              .where(
+                and(
+                  inArray(schema.documentShares.resourceId, visibleDocumentIds),
+                  or(...principalClauses),
+                ),
+              )
+          : Promise.resolve([] as { resourceId: string; role: ShareRole }[]),
+        db
+          .select()
+          .from(schema.contentDatabases)
+          .where(
+            and(
+              inArray(schema.contentDatabases.documentId, visibleDocumentIds),
+              isNull(schema.contentDatabases.deletedAt),
+            ),
+          ),
+        db
           .select({
-            resourceId: schema.documentShares.resourceId,
-            role: schema.documentShares.role,
+            item: schema.contentDatabaseItems,
+            database: schema.contentDatabases,
           })
-          .from(schema.documentShares)
+          .from(schema.contentDatabaseItems)
+          .innerJoin(
+            schema.contentDatabases,
+            eq(
+              schema.contentDatabases.id,
+              schema.contentDatabaseItems.databaseId,
+            ),
+          )
           .where(
             and(
               inArray(
-                schema.documentShares.resourceId,
-                documents.map((d) => d.id),
+                schema.contentDatabaseItems.documentId,
+                visibleDocumentIds,
               ),
-              or(...principalClauses),
+              isNull(schema.contentDatabases.deletedAt),
             ),
-          );
-
-        for (const row of shareRows) {
-          shareRoleByDocumentId.set(
-            row.resourceId,
-            strongerRole(
-              shareRoleByDocumentId.get(row.resourceId) ?? null,
-              row.role,
+          ),
+        db
+          .select({
+            id: schema.contentDatabases.id,
+            documentId: schema.contentDatabases.documentId,
+          })
+          .from(schema.contentDatabases)
+          .where(
+            and(
+              inArray(schema.contentDatabases.documentId, visibleDocumentIds),
+              isNotNull(schema.contentDatabases.deletedAt),
             ),
-          );
-        }
-      }
-    }
+          ),
+      ]);
 
-    const databases =
-      documents.length > 0
-        ? await db
-            .select()
-            .from(schema.contentDatabases)
-            .where(
-              and(
-                inArray(
-                  schema.contentDatabases.documentId,
-                  documents.map((d) => d.id),
-                ),
-                isNull(schema.contentDatabases.deletedAt),
-              ),
-            )
-        : [];
-    const databaseByDocumentId = new Map(
-      databases.map((database) => [database.documentId, database]),
-    );
-    const databaseMemberships =
-      documents.length > 0
-        ? await db
-            .select({
-              item: schema.contentDatabaseItems,
-              database: schema.contentDatabases,
-            })
-            .from(schema.contentDatabaseItems)
-            .innerJoin(
-              schema.contentDatabases,
-              eq(
-                schema.contentDatabases.id,
-                schema.contentDatabaseItems.databaseId,
-              ),
-            )
-            .where(
-              and(
-                inArray(
-                  schema.contentDatabaseItems.documentId,
-                  documents.map((d) => d.id),
-                ),
-                isNull(schema.contentDatabases.deletedAt),
-              ),
-            )
-        : [];
-    const databaseMembershipByDocumentId = new Map(
-      databaseMemberships.map((row) => [row.item.documentId, row]),
-    );
+      for (const link of notionLinks) {
+        notionPageIdByDocumentId.set(link.documentId, link.remotePageId);
+      }
 
-    const softDeletedDocumentIds = new Set<string>();
-    if (documents.length > 0) {
-      const visibleDocumentIds = documents.map((d) => d.id);
-      const softDeletedDatabases = await db
-        .select({
-          id: schema.contentDatabases.id,
-          documentId: schema.contentDatabases.documentId,
-        })
-        .from(schema.contentDatabases)
-        .where(
-          and(
-            inArray(schema.contentDatabases.documentId, visibleDocumentIds),
-            isNotNull(schema.contentDatabases.deletedAt),
+      for (const row of shareRows) {
+        shareRoleByDocumentId.set(
+          row.resourceId,
+          strongerRole(
+            shareRoleByDocumentId.get(row.resourceId) ?? null,
+            row.role,
           ),
         );
+      }
+
+      for (const database of databases) {
+        databaseByDocumentId.set(database.documentId, database);
+      }
+
+      for (const row of databaseMemberships) {
+        databaseMembershipByDocumentId.set(row.item.documentId, row);
+      }
 
       for (const database of softDeletedDatabases) {
         softDeletedDocumentIds.add(database.documentId);

package/corpus/templates/content/actions/search-documents.ts

@@ -17,6 +17,12 @@ function escapeLike(s: string): string {
   return s.replace(/([\\%_])/g, "\\$1");
 }
 
+// `content` here may be a bounded preview (see the `contentPreview`
+// projection below) rather than the full document body. If the query match
+// falls outside the preview window (a deeper match in the full doc, which the
+// SQL LIKE filter already confirmed exists), `indexOf` simply misses and we
+// fall back to a beginning-of-document snippet — the same behavior as the
+// no-match case. The row is still returned either way.
 function makeSnippet(content: string, query: string, radius = 120) {
   const compact = content.replace(/\s+/g, " ").trim();
   if (!compact) return "";
@@ -75,13 +81,23 @@ export default defineAction({
     const db = getDb();
     const pattern = `%${escapeLike(query)}%`;
 
+    // Project a bounded preview of `content` instead of the full column:
+    // document bodies can be multi-MB, and this action only returns a short
+    // snippet (use get-document for full content). 5000 chars is generous
+    // headroom for `makeSnippet`'s 120-char radius even when the match is
+    // deep-ish into the doc, while the true length still comes from SQL
+    // `length()` rather than reading `.length` off a truncated string.
+    // Mirrors the `substr`/`length` projection style in list-documents.ts.
+    // Both `substr` and `length` work identically on SQLite/libsql and
+    // Postgres.
     const docs = await db
       .select({
         id: schema.documents.id,
         parentId: schema.documents.parentId,
         title: schema.documents.title,
         icon: schema.documents.icon,
-        content: schema.documents.content,
+        contentPreview: sql<string>`substr(${schema.documents.content}, 1, 5000)`,
+        contentLength: sql<number>`length(${schema.documents.content})`,
         hideFromSearch: schema.documents.hideFromSearch,
         updatedAt: schema.documents.updatedAt,
       })
@@ -102,8 +118,8 @@ export default defineAction({
         parentId: doc.parentId,
         title: doc.title,
         icon: doc.icon,
-        snippet: makeSnippet(doc.content, query),
-        contentLength: doc.content.length,
+        snippet: makeSnippet(doc.contentPreview, query),
+        contentLength: Number(doc.contentLength) || 0,
         hideFromSearch: parseDocumentHideFromSearch(doc.hideFromSearch),
         updatedAt: doc.updatedAt,
       })),