@agent-native/core-corpus 0.212.1 → 0.214.0

package/corpus/templates/content/actions/_github-source.ts

@@ -31,6 +31,7 @@ import {
   GITHUB_SOURCE_MAX_IMAGE_BYTES,
   GITHUB_SOURCE_TYPE,
   githubBranchCandidates,
+  githubFieldExamples,
   githubFieldPropertyInput,
   githubFolderLabel,
   githubFolderWebUrl,
@@ -41,6 +42,7 @@ import {
   parseGitHubFolderUrl,
   type GitHubFieldType,
   type GitHubFolderLocation,
+  type GitHubFolderPreview,
   type GitHubSourceProblemCode,
   type GitHubSourceSkipReason,
 } from "../shared/github-source.js";
@@ -78,6 +80,7 @@ import {
   githubSourceMetadataRecord,
   readGitHubSourceState,
   rememberedSkips,
+  type GitHubReadSkip,
   type GitHubSourceState,
 } from "./_github-source-metadata.js";
 import {
@@ -201,6 +204,7 @@ const PROBLEM_STATUS: Record<GitHubSourceProblemCode, number> = {
   rate_limited: 429,
   too_large: 413,
   unavailable: 502,
+  sync_failed: 500,
   storage_unavailable: 503,
 };
 
@@ -337,37 +341,6 @@ function frontmatterValues(
   return draft.frontmatter.unmapped ?? {};
 }
 
-export interface GitHubFolderPreview {
-  location: GitHubFolderLocation;
-  label: string;
-  folderUrl: string;
-  commitSha: string;
-  markdownFiles: number;
-  skipped: Array<{ path: string; reason: GitHubSourceSkipReason }>;
-  /** Columns the frontmatter would become, with the type each would get. */
-  fields: Array<{ key: string; type: GitHubFieldType; examples: string[] }>;
-  sample: Array<{ path: string; title: string }>;
-  /** How many files the field types were read from. */
-  sampledFiles: number;
-  /** Image files in the folder, which pages show from Content's file storage. */
-  images: number;
-  /** Whether file storage is set up to hold those images. */
-  storageReady: boolean;
-}
-
-function exampleText(value: unknown): string {
-  if (typeof value === "string") return value.slice(0, 80);
-  if (Array.isArray(value)) {
-    return value
-      .map((entry) =>
-        typeof entry === "string" ? entry : JSON.stringify(entry),
-      )
-      .join(", ")
-      .slice(0, 80);
-  }
-  return JSON.stringify(value).slice(0, 80);
-}
-
 /**
  * Reports what linking the folder would create without creating anything:
  * how many files become rows, which are skipped and why, and the columns the
@@ -439,10 +412,7 @@ export async function previewGitHubFolder(
     fields: [...valuesByKey].map(([key, values]) => ({
       key,
       type: inferGitHubFieldType(key, values),
-      examples: values
-        .filter((value) => value !== null && value !== "")
-        .slice(0, 3)
-        .map(exampleText),
+      examples: githubFieldExamples(values),
     })),
     sample,
     sampledFiles: sample.length,
@@ -788,6 +758,26 @@ function emptyResult(
   };
 }
 
+/**
+ * The problem a sync records when it fails for a reason it can't name, like a
+ * database error. The GitHub client names its own failures, a dropped
+ * connection included, so this one isn't GitHub's.
+ */
+function unexpectedProblem(
+  at: string,
+): NonNullable<GitHubSourceState["problem"]> {
+  return {
+    code: "sync_failed",
+    message: "The sync failed. The background sync tries again.",
+    at,
+  };
+}
+
+/** A sync that failed before it could record why. */
+export function unexpectedSyncFailure(at: string): GitHubSyncResult {
+  return emptyResult("error", null, unexpectedProblem(at));
+}
+
 /** The status problem for images a sync couldn't store, if there are any. */
 function storageProblem(
   imagesWaiting: number,
@@ -1643,6 +1633,11 @@ export interface SyncGitHubSourceOptions {
   force?: boolean;
   /** Rethrow a missing GitHub connection so an interactive caller can offer to connect. */
   throwConnectionErrors?: boolean;
+  /**
+   * Rethrow a failure the sync can't name, like a database error, once it's
+   * on the link, for a caller that reports failures as errors.
+   */
+  throwUnexpectedErrors?: boolean;
   /** Ends the sync early, as its time budget would, such as when the server stops. */
   signal?: AbortSignal;
 }
@@ -1704,6 +1699,19 @@ export async function syncGitHubSource(
     () => budget.abort(),
     Math.max(0, deadline - Date.now()),
   );
+  // Counts what this run writes. A run that fails partway still reports the
+  // rows it changed first, so readers hear about them.
+  const result = emptyResult("synced", null);
+  // Files this run read and had to skip. A run that fails partway keeps
+  // them too, so the next one doesn't read them again.
+  let readSkipsSoFar: GitHubReadSkip[] | null = null;
+  const failed = (problem: GitHubSourceState["problem"]): GitHubSyncResult => ({
+    ...emptyResult("error", state.pendingCommitSha, problem),
+    created: result.created,
+    updated: result.updated,
+    renamed: result.renamed,
+    trashed: result.trashed,
+  });
   const endBudget = () => budget.abort();
   if (options.signal?.aborted) endBudget();
   options.signal?.addEventListener("abort", endBudget, { once: true });
@@ -1862,7 +1870,18 @@ export async function syncGitHubSource(
       run.documentIdByPath.set(folderRelative(run, path), row.documentId);
     }
 
+    // A file skipped once it was read is skipped again while its blob is the
+    // same, so it isn't read again. It has no row to record that on.
+    const readSkipsByPath = new Map(
+      state.readSkips.map((skip) => [skip.path, skip] as const),
+    );
+    const stillSkipped: GitHubReadSkip[] = [];
     const work = planned.filter((file) => {
+      const skip = readSkipsByPath.get(file.entry.path);
+      if (skip?.blobSha === file.entry.sha) {
+        stillSkipped.push(skip);
+        return false;
+      }
       const previous = file.row ? rowState(file.row) : null;
       return (
         !previous ||
@@ -1880,9 +1899,10 @@ export async function syncGitHubSource(
       (a, b) =>
         (cutShort.get(a.repoPath) ?? 0) - (cutShort.get(b.repoPath) ?? 0),
     );
-    const result = emptyResult("synced", head.sha);
-    result.unchanged = planned.length - work.length;
-    const skippedFiles = [...files.skipped];
+    result.commitSha = head.sha;
+    result.unchanged = planned.length - work.length - stillSkipped.length;
+    const readSkips: GitHubReadSkip[] = [...stillSkipped];
+    readSkipsSoFar = readSkips;
     let done = 0;
     let cutShortPath: string | null = null;
     for (const file of work) {
@@ -1923,7 +1943,11 @@ export async function syncGitHubSource(
       }
       if (typeof outcome === "string") result[outcome] += 1;
       else
-        skippedFiles.push({ path: file.entry.path, reason: outcome.skipped });
+        readSkips.push({
+          path: file.entry.path,
+          reason: outcome.skipped,
+          blobSha: file.entry.sha,
+        });
       done += 1;
     }
     const remainingFiles = work.length - done;
@@ -1957,40 +1981,55 @@ export async function syncGitHubSource(
     // moved it for two minutes. One that got nowhere waits its turn, as a
     // failed check does.
     const countsAsCheck = complete || (done === 0 && result.trashed === 0);
-    const skipped = rememberedSkips(
-      skippedFiles.sort((a, b) => (a.path < b.path ? -1 : 1)),
-    );
+    const skippedFiles = [...files.skipped, ...readSkips];
+    const skipped = rememberedSkips(skippedFiles);
     await release(
-      (current) => ({
-        ...current,
-        checkedAt: now,
-        problem: result.problem,
-        files: files.markdown.length,
-        skipped,
-        imagesWaiting: result.imagesWaiting,
-        ...(complete
-          ? {
-              syncedCommitSha: head.sha,
-              syncedAt: new Date().toISOString(),
-              headEtag: head.etag,
-              pendingCommitSha: null,
-              progress: null,
-              cutShort: [],
-            }
-          : {
-              pendingCommitSha: head.sha,
-              cutShort: cutShortPath
-                ? [
-                    ...current.cutShort.filter((path) => path !== cutShortPath),
-                    cutShortPath,
-                  ]
-                : current.cutShort,
-              progress: {
-                done: work.length - remainingFiles + result.trashed,
-                total: work.length + result.trashed + gone.size,
-              },
-            }),
-      }),
+      (current) => {
+        // A run that carries on a push sees only the files left, so it keeps
+        // the total the push started with and counts what's left against it.
+        // The count never goes back.
+        const before =
+          current.pendingCommitSha === head.sha ? current.progress : null;
+        const total = before?.total ?? work.length + result.trashed + gone.size;
+        const progress = {
+          done: Math.min(
+            total,
+            Math.max(before?.done ?? 0, total - result.remaining),
+          ),
+          total,
+        };
+        return {
+          ...current,
+          checkedAt: now,
+          problem: result.problem,
+          files: files.markdown.length,
+          skipped,
+          skippedCount: skippedFiles.length,
+          readSkips,
+          imagesWaiting: result.imagesWaiting,
+          ...(complete
+            ? {
+                syncedCommitSha: head.sha,
+                syncedAt: new Date().toISOString(),
+                headEtag: head.etag,
+                pendingCommitSha: null,
+                progress: null,
+                cutShort: [],
+              }
+            : {
+                pendingCommitSha: head.sha,
+                cutShort: cutShortPath
+                  ? [
+                      ...current.cutShort.filter(
+                        (path) => path !== cutShortPath,
+                      ),
+                      cutShortPath,
+                    ]
+                  : current.cutShort,
+                progress,
+              }),
+        };
+      },
       () => ({
         syncState: complete ? "linked" : "refreshing",
         freshness: complete ? "fresh" : "stale",
@@ -2017,16 +2056,33 @@ export async function syncGitHubSource(
           lastRefreshedAt: now,
         }),
       ).catch(warnLeaseNotReleased);
-      return emptyResult("error", state.pendingCommitSha, {
+      return failed({
         code: "unavailable",
         message:
           "GitHub didn't answer in time. The background sync tries again.",
         at: now,
       });
     }
-    const problem = problemFrom(error, now);
+    // A contract error, like the link going away mid-sync, is the caller's to
+    // see. Anything else the sync can't name is still a failed check: the
+    // status line shows it and the next sync tries again.
+    const named = problemFrom(error, now);
+    const problem =
+      named ??
+      (error instanceof ActionContractError ? null : unexpectedProblem(now));
+    if (!named && problem) {
+      console.warn(
+        "[github-source] sync failed:",
+        error instanceof Error ? error.message : String(error),
+      );
+    }
     await release(
-      (current) => ({ ...current, checkedAt: now, problem }),
+      (current) => ({
+        ...current,
+        checkedAt: now,
+        problem,
+        ...(readSkipsSoFar ? { readSkips: readSkipsSoFar } : {}),
+      }),
       () => ({
         syncState: "error",
         freshness: "stale",
@@ -2038,7 +2094,8 @@ export async function syncGitHubSource(
     if (problem.code === "connection" && options.throwConnectionErrors) {
       throw error;
     }
-    return emptyResult("error", state.pendingCommitSha, problem);
+    if (!named && options.throwUnexpectedErrors) throw error;
+    return failed(problem);
   } finally {
     clearTimeout(budgetTimer);
     options.signal?.removeEventListener("abort", endBudget);
@@ -2078,6 +2135,8 @@ export async function createGitHubSourceLink(args: {
     cutShort: [],
     files: preview.markdownFiles,
     skipped: rememberedSkips(preview.skipped),
+    skippedCount: preview.skipped.length,
+    readSkips: [],
     problem: null,
     claim: null,
   };
@@ -2161,6 +2220,11 @@ export function githubPreviewSummary(preview: GitHubFolderPreview) {
         ),
       )}.`
     : " It has no frontmatter columns.";
+  // Columns come from the first files only; later ones can differ.
+  const sampled =
+    preview.sampledFiles < preview.markdownFiles
+      ? ` That frontmatter is read from ${preview.sampledFiles} of its ${preview.markdownFiles} files, so the others can add columns or change their types.`
+      : "";
   const skipped = preview.skipped.length
     ? ` ${preview.skipped.length} skipped: ${listWithMore(
         preview.skipped.map(
@@ -2172,7 +2236,7 @@ export function githubPreviewSummary(preview: GitHubFolderPreview) {
     preview.images > 0 && !preview.storageReady
       ? ` Its ${preview.images === 1 ? "image won't" : `${preview.images} images won't`} show until file storage is set up in Settings → File uploads.`
       : "";
-  return `${preview.label} at ${preview.commitSha.slice(0, 7)} has ${preview.markdownFiles} Markdown ${preview.markdownFiles === 1 ? "file" : "files"}.${columns}${skipped}${images}`;
+  return `${preview.label} at ${preview.commitSha.slice(0, 7)} has ${preview.markdownFiles} Markdown ${preview.markdownFiles === 1 ? "file" : "files"}.${columns}${sampled}${skipped}${images}`;
 }
 
 export function githubSyncSummary(label: string, result: GitHubSyncResult) {

package/corpus/templates/content/actions/_property-utils.ts

@@ -30,6 +30,7 @@ import type {
   DocumentProperty,
 } from "../shared/api.js";
 import { blocksFieldId } from "../shared/blocks-field-identity.js";
+import { contentDatabaseItemLayoutField } from "../shared/database-item-layout.js";
 import {
   DEFAULT_BLOCKS_FIELD_NAME,
   defaultPropertyOptions,
@@ -398,6 +399,7 @@ function defaultDatabaseView(
     rowDensity: normalizeDatabaseRowDensity(values.rowDensity),
     openPagesIn: normalizeDatabaseOpenPagesIn(values.openPagesIn),
     formQuestions: normalizeDatabaseFormQuestions(values.formQuestions),
+    ...contentDatabaseItemLayoutField(values.itemLayout),
   };
 }
 
@@ -460,6 +462,7 @@ function normalizeDatabaseView(value: unknown): ContentDatabaseView | null {
     rowDensity: normalizeDatabaseRowDensity(view.rowDensity),
     openPagesIn: normalizeDatabaseOpenPagesIn(view.openPagesIn),
     formQuestions: normalizeDatabaseFormQuestions(view.formQuestions),
+    ...contentDatabaseItemLayoutField(view.itemLayout),
   };
 }
 

package/corpus/templates/content/actions/check-github-connection.ts

@@ -0,0 +1,15 @@
+import { defineAction } from "@agent-native/core/action";
+import { z } from "zod";
+
+import { readGitHubConnected } from "../server/lib/github-source-client.js";
+
+export default defineAction({
+  description:
+    "Check whether the current user has a GitHub connection Content can read folders through, before linking one. Reads no repository. `connected` is null when GitHub couldn't be asked.",
+  schema: z.object({}).strict(),
+  http: { method: "GET" },
+  readOnly: true,
+  run: async (): Promise<{ connected: boolean | null }> => ({
+    connected: await readGitHubConnected(),
+  }),
+});

package/corpus/templates/content/actions/create-github-collection.ts

@@ -0,0 +1,180 @@
+import { ActionContractError, defineAction } from "@agent-native/core/action";
+import { getRequestUserEmail } from "@agent-native/core/server/request-context";
+import { assertAccess } from "@agent-native/core/sharing";
+import { z } from "zod";
+
+import { documentChangeResource } from "../server/lib/document-change-resource.js";
+import { githubFolderName } from "../shared/github-source.js";
+import { resolveContentSpaceAccess } from "./_content-space-access.js";
+import type { DatabaseSetupReceipt } from "./_database-setup-mutation.js";
+import { resolveDatabaseForSourceMutation } from "./_database-source-utils.js";
+import {
+  githubActionError,
+  githubPreviewSummary,
+  previewGitHubFolder,
+  readLinkedGitHubSource,
+  resolveGitHubFolder,
+} from "./_github-source.js";
+import { nanoid, serializeDatabase } from "./_property-utils.js";
+import createContentDatabase from "./create-content-database.js";
+import deleteContentDatabase from "./delete-content-database.js";
+import {
+  bringInGitHubLink,
+  linkPreviewedGitHubFolder,
+} from "./link-github-folder.js";
+
+const createGitHubCollectionSchema = z
+  .object({
+    url: z
+      .string()
+      .min(1)
+      .describe(
+        "GitHub folder link, like https://github.com/owner/repo/tree/main/docs, or owner/repo for the whole repository",
+      ),
+    spaceId: z
+      .string()
+      .min(1)
+      .describe("Exact Content space ID for the new collection"),
+    parentId: z
+      .string()
+      .min(1)
+      .nullish()
+      .describe("Page to put the collection under; leave out for the space"),
+    connectionId: z
+      .string()
+      .min(1)
+      .optional()
+      .describe(
+        "GitHub connection to read through when there is more than one",
+      ),
+    dryRun: z
+      .boolean()
+      .optional()
+      .describe("Preview only: read the folder without making the collection"),
+    idempotencyKey: z
+      .string()
+      .min(1)
+      .max(200)
+      .optional()
+      .describe(
+        "Creation intent key; reuse it unchanged after a lost response",
+      ),
+  })
+  .strict();
+
+export default defineAction({
+  description:
+    "Make a new collection from a GitHub folder, named after the folder and linked to it view only: each Markdown file becomes a row and pushes to the branch arrive on their own. Run with dryRun first to see the files, skipped files and columns without making anything.",
+  mcpTool: true,
+  // It only makes a new collection, so it never replaces anyone's content.
+  mcpAnnotations: {
+    readOnlyHint: false,
+    destructiveHint: false,
+    openWorldHint: true,
+  },
+  agentInputSchema: createGitHubCollectionSchema,
+  schema: createGitHubCollectionSchema,
+  run: async (args) => {
+    const userEmail = getRequestUserEmail();
+    if (!userEmail) {
+      throw new ActionContractError("Sign in to link a GitHub folder.", {
+        errorCode: "AUTH_REQUIRED",
+        statusCode: 401,
+      });
+    }
+    // The same checks the new collection gets, so a preview doesn't promise
+    // a collection the person can't make.
+    await resolveContentSpaceAccess(args.spaceId, "contributor");
+    if (args.parentId) await assertAccess("document", args.parentId, "editor");
+
+    const connectionId = args.connectionId ?? null;
+    let resolved: Awaited<ReturnType<typeof resolveGitHubFolder>>;
+    let preview: Awaited<ReturnType<typeof previewGitHubFolder>>;
+    try {
+      resolved = await resolveGitHubFolder({ url: args.url, connectionId });
+      preview = await previewGitHubFolder(resolved, connectionId);
+    } catch (error) {
+      throw githubActionError(error);
+    }
+    const title = githubFolderName(preview.location);
+
+    if (args.dryRun) {
+      return {
+        dryRun: true as const,
+        preview,
+        title,
+        summary: `${githubPreviewSummary(preview)} Run again without dryRun to make the "${title}" collection and link it.`,
+      };
+    }
+
+    const created = await createContentDatabase.run({
+      spaceId: args.spaceId,
+      parentId: args.parentId ?? null,
+      title,
+      idempotencyKey: args.idempotencyKey ?? `github-collection-${nanoid()}`,
+    });
+    const database = await resolveDatabaseForSourceMutation({
+      databaseId: created.database.id,
+    });
+    if (!database) {
+      throw new ActionContractError("The new collection can't be found.", {
+        errorCode: "DATABASE_NOT_FOUND",
+        statusCode: 404,
+      });
+    }
+    // A retry after a lost response gets back the collection the first try
+    // made, which may be linked already.
+    const { receipt } = created as { receipt?: DatabaseSetupReceipt };
+    const madeEarlier = receipt?.idempotency.result === "replayed";
+    const earlierLink = madeEarlier
+      ? await readLinkedGitHubSource(database.id)
+      : null;
+    if (earlierLink && earlierLink.sourceName === preview.label) {
+      return {
+        dryRun: false as const,
+        database: serializeDatabase(database),
+        preview,
+        title,
+        ...(await bringInGitHubLink({
+          database,
+          preview,
+          sourceId: earlierLink.id,
+        })),
+      };
+    }
+    let linked: Awaited<ReturnType<typeof linkPreviewedGitHubFolder>>;
+    try {
+      linked = await linkPreviewedGitHubFolder({
+        database,
+        resolved,
+        preview,
+        connectionId,
+        userEmail,
+      });
+    } catch (error) {
+      // The collection was made for this folder, so it doesn't stay behind
+      // empty when the link can't be saved. One an earlier try made is left
+      // alone.
+      if (madeEarlier) throw error;
+      try {
+        await deleteContentDatabase.run({ databaseId: database.id });
+      } catch (cleanupError) {
+        console.warn(
+          "[create-github-collection] could not remove the new collection",
+          cleanupError,
+        );
+      }
+      throw error;
+    }
+    return {
+      dryRun: false as const,
+      database: serializeDatabase(database),
+      preview,
+      title,
+      ...linked,
+    };
+  },
+  // Everyone reading the new collection gets its rows, not just the caller.
+  changeResource: (_input, result) =>
+    result.dryRun ? null : documentChangeResource(result.database.documentId),
+});

package/corpus/templates/content/actions/disconnect-content-database-source.ts

@@ -4,6 +4,7 @@ import { and, eq, inArray } from "drizzle-orm";
 import { z } from "zod";
 
 import { getDb, schema } from "../server/db/index.js";
+import { documentChangeResource } from "../server/lib/document-change-resource.js";
 import type {
   ContentDatabaseResponse,
   DisconnectContentDatabaseSourceRequest,
@@ -125,4 +126,8 @@ export default defineAction({
 
     return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 });
   },
+  // Everyone reading the collection sees the source go, and a GitHub
+  // collection's pages become editable for them too.
+  changeResource: (_input, result) =>
+    documentChangeResource(result.database.documentId),
 });