@gmickel/gno 1.24.0 → 1.26.0

package/src/core/browser-clip-provenance.ts

@@ -0,0 +1,139 @@
+import { z } from "zod";
+
+export const BROWSER_CLIP_SCHEMA_VERSION = "1.0" as const;
+export const BROWSER_CLIP_MAX_BYTES = 512 * 1024;
+
+export const BROWSER_CLIP_WARNING_CODES = [
+  "authenticated_visible_content",
+  "canonical_url_differs",
+  "edited_content",
+  "line_endings_normalized",
+  "reader_partial",
+  "selection_truncated",
+  "spa_snapshot",
+  "unicode_normalized",
+] as const;
+
+export const BROWSER_CLIP_HTTP_URL_PATTERN =
+  "^[Hh][Tt][Tt][Pp][Ss]?://(?:(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\\.)*[A-Za-z](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?::(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?(?:(?:/(?:[A-Za-z0-9._~!$&'()*+,;=:@-]|%[0-9A-Fa-f]{2})*)*)(?:\\?(?:[A-Za-z0-9._~!$&'()*+,;=:@/?-]|%[0-9A-Fa-f]{2})*)?(?:#(?:[A-Za-z0-9._~!$&'()*+,;=:@/?-]|%[0-9A-Fa-f]{2})*)?$";
+
+const hasDisallowedControlCharacter = (value: string): boolean => {
+  for (const character of value) {
+    const codePoint = character.codePointAt(0);
+    if (
+      codePoint !== undefined &&
+      (codePoint <= 8 ||
+        codePoint === 11 ||
+        codePoint === 12 ||
+        (codePoint >= 14 && codePoint <= 31) ||
+        (codePoint >= 127 && codePoint <= 159))
+    ) {
+      return true;
+    }
+  }
+  return false;
+};
+
+export const findDisallowedBrowserClipControlPath = (
+  value: unknown,
+  path: Array<string | number> = []
+): Array<string | number> | null => {
+  if (typeof value === "string") {
+    return hasDisallowedControlCharacter(value) ? path : null;
+  }
+  if (Array.isArray(value)) {
+    for (const [index, child] of value.entries()) {
+      const match = findDisallowedBrowserClipControlPath(child, [
+        ...path,
+        index,
+      ]);
+      if (match !== null) return match;
+    }
+    return null;
+  }
+  if (value !== null && typeof value === "object") {
+    for (const [key, child] of Object.entries(value)) {
+      const match = findDisallowedBrowserClipControlPath(child, [...path, key]);
+      if (match !== null) return match;
+    }
+  }
+  return null;
+};
+
+export const browserClipHttpUrlSchema = z
+  .string()
+  .max(4096)
+  .regex(
+    new RegExp(BROWSER_CLIP_HTTP_URL_PATTERN, "u"),
+    "URL must use the browser clip HTTP(S) URL subset"
+  );
+
+const publishedAtSchema = z.union([
+  z.string().date(),
+  z.string().datetime({ offset: true }),
+]);
+
+const browserSchema = z
+  .object({
+    name: z.string().min(1).max(128),
+    version: z.string().max(128).nullable(),
+    platform: z.string().max(128).nullable(),
+  })
+  .strict();
+
+const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/);
+
+export const browserClipProvenanceSchema = z
+  .object({
+    schemaVersion: z.literal(BROWSER_CLIP_SCHEMA_VERSION),
+    mode: z.enum(["selection", "reader"]),
+    sourceUrl: browserClipHttpUrlSchema,
+    canonicalUrl: browserClipHttpUrlSchema.nullable(),
+    title: z.string().min(1).max(2048),
+    author: z.string().max(1024).nullable(),
+    site: z.string().max(1024).nullable(),
+    publishedAt: publishedAtSchema.nullable(),
+    observedAt: z.string().datetime({ offset: true }),
+    capturedAt: z.string().datetime({ offset: true }),
+    extractionHash: sha256Schema,
+    finalBodyHash: sha256Schema,
+    clipIdentity: sha256Schema,
+    previewDigest: sha256Schema,
+    exactSelection: z.string().min(1).max(BROWSER_CLIP_MAX_BYTES).nullable(),
+    extractionWarnings: z.array(z.enum(BROWSER_CLIP_WARNING_CODES)).max(16),
+    browser: browserSchema,
+  })
+  .strict()
+  .superRefine((value, context) => {
+    const controlPath = findDisallowedBrowserClipControlPath(value);
+    if (controlPath !== null) {
+      context.addIssue({
+        code: "custom",
+        message: "Browser clip text cannot contain C0 or C1 control characters",
+        path: controlPath,
+      });
+    }
+    if (
+      new Set(value.extractionWarnings).size !== value.extractionWarnings.length
+    ) {
+      context.addIssue({
+        code: "custom",
+        message: "Warning codes must be unique",
+        path: ["extractionWarnings"],
+      });
+    }
+    if (
+      (value.mode === "selection" && value.exactSelection === null) ||
+      (value.mode === "reader" && value.exactSelection !== null)
+    ) {
+      context.addIssue({
+        code: "custom",
+        message: "Exact selection must match the extraction mode",
+        path: ["exactSelection"],
+      });
+    }
+  });
+
+export type BrowserClipProvenance = z.infer<typeof browserClipProvenanceSchema>;
+export type BrowserClipWarningCode =
+  (typeof BROWSER_CLIP_WARNING_CODES)[number];

package/src/core/browser-clip.ts

@@ -0,0 +1,473 @@
+import { z } from "zod";
+
+import { canonicalize } from "../converters/canonicalize";
+import {
+  BROWSER_CLIP_MAX_BYTES,
+  BROWSER_CLIP_SCHEMA_VERSION,
+  BROWSER_CLIP_WARNING_CODES,
+  browserClipHttpUrlSchema,
+  findDisallowedBrowserClipControlPath,
+  type BrowserClipProvenance,
+  type BrowserClipWarningCode,
+} from "./browser-clip-provenance";
+import {
+  hashCaptureContent,
+  type CaptureInput,
+  type CaptureSource,
+} from "./capture";
+
+export {
+  BROWSER_CLIP_MAX_BYTES,
+  BROWSER_CLIP_SCHEMA_VERSION,
+} from "./browser-clip-provenance";
+export type {
+  BrowserClipProvenance,
+  BrowserClipWarningCode,
+} from "./browser-clip-provenance";
+
+const sha256 = (value: string): string =>
+  new Bun.CryptoHasher("sha256").update(value).digest("hex");
+
+const stableJsonValue = (value: unknown): unknown => {
+  if (Array.isArray(value)) return value.map(stableJsonValue);
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+        .map(([key, child]) => [key, stableJsonValue(child)])
+    );
+  }
+  return value;
+};
+
+const stableJson = (value: unknown): string =>
+  JSON.stringify(stableJsonValue(value));
+
+const strictDateTime = z.string().datetime({ offset: true });
+const strictPublishedDate = z
+  .union([z.string().date(), z.string().datetime({ offset: true })])
+  .nullable();
+
+const safeInlineText = z.string().min(1).max(32_768);
+const inlineNodeSchema = z.discriminatedUnion("type", [
+  z.object({ type: z.literal("text"), text: safeInlineText }).strict(),
+  z
+    .object({
+      type: z.literal("link"),
+      text: safeInlineText,
+      href: browserClipHttpUrlSchema,
+    })
+    .strict(),
+]);
+const inlineContentSchema = z.array(inlineNodeSchema).min(1).max(256);
+
+const readerBlockSchema = z.discriminatedUnion("type", [
+  z
+    .object({ type: z.literal("paragraph"), content: inlineContentSchema })
+    .strict(),
+  z
+    .object({
+      type: z.literal("heading"),
+      level: z.number().int().min(1).max(6),
+      content: inlineContentSchema,
+    })
+    .strict(),
+  z.object({ type: z.literal("quote"), content: inlineContentSchema }).strict(),
+  z
+    .object({
+      type: z.literal("list"),
+      ordered: z.boolean(),
+      items: z.array(inlineContentSchema).min(1).max(256),
+    })
+    .strict(),
+  z
+    .object({
+      type: z.literal("code"),
+      language: z
+        .string()
+        .regex(/^[A-Za-z0-9_+-]{1,32}$/)
+        .nullable(),
+      text: z.string().max(131_072),
+    })
+    .strict(),
+  z.object({ type: z.literal("horizontal_rule") }).strict(),
+]);
+
+const commonPayload = z
+  .object({
+    schemaVersion: z.literal(BROWSER_CLIP_SCHEMA_VERSION),
+    sourceUrl: browserClipHttpUrlSchema,
+    canonicalUrl: browserClipHttpUrlSchema.nullable(),
+    title: z.string().min(1).max(2048),
+    author: z.string().max(1024).nullable(),
+    site: z.string().max(1024).nullable(),
+    publishedAt: strictPublishedDate,
+    observedAt: strictDateTime,
+    browser: z
+      .object({
+        name: z.string().min(1).max(128),
+        version: z.string().max(128).nullable(),
+        platform: z.string().max(128).nullable(),
+      })
+      .strict(),
+    extraction: z
+      .object({
+        visibility: z.literal("user_visible"),
+        authenticated: z.boolean(),
+        extractorVersion: z.string().min(1).max(128),
+        warnings: z.array(z.enum(BROWSER_CLIP_WARNING_CODES)).max(16),
+      })
+      .strict(),
+    destination: z
+      .object({
+        collection: z.string().min(1).max(128),
+        relPath: z.string().min(1).max(2048).nullable(),
+        folderPath: z.string().min(1).max(2048).nullable(),
+        collisionPolicy: z.enum([
+          "error",
+          "open_existing",
+          "create_with_suffix",
+        ]),
+      })
+      .strict(),
+    tags: z.array(z.string().min(1).max(256)).max(128),
+    note: z.string().max(32_768).nullable(),
+  })
+  .strict();
+
+const selectionPayload = commonPayload
+  .extend({
+    mode: z.literal("selection"),
+    selection: z
+      .object({
+        exactText: z.string().min(1).max(BROWSER_CLIP_MAX_BYTES),
+        editedMarkdown: z.string().max(BROWSER_CLIP_MAX_BYTES).nullable(),
+      })
+      .strict(),
+  })
+  .strict();
+
+const readerPayload = commonPayload
+  .extend({
+    mode: z.literal("reader"),
+    reader: z
+      .object({
+        blocks: z.array(readerBlockSchema).min(1).max(4096),
+        editedMarkdown: z.string().max(BROWSER_CLIP_MAX_BYTES).nullable(),
+      })
+      .strict(),
+  })
+  .strict();
+
+export const browserClipPayloadSchema = z
+  .discriminatedUnion("mode", [selectionPayload, readerPayload])
+  .superRefine((value, context) => {
+    const controlPath = findDisallowedBrowserClipControlPath(value);
+    if (controlPath !== null) {
+      context.addIssue({
+        code: "custom",
+        message: "Browser clip text cannot contain C0 or C1 control characters",
+        path: controlPath,
+      });
+    }
+    if (
+      new TextEncoder().encode(stableJson(value)).byteLength >
+      BROWSER_CLIP_MAX_BYTES
+    ) {
+      context.addIssue({
+        code: "custom",
+        message: "Browser clip payload is too large",
+      });
+    }
+    if (
+      new Set(value.extraction.warnings).size !==
+      value.extraction.warnings.length
+    ) {
+      context.addIssue({
+        code: "custom",
+        message: "Warning codes must be unique",
+      });
+    }
+    if (
+      value.destination.relPath !== null &&
+      value.destination.folderPath !== null
+    ) {
+      context.addIssue({
+        code: "custom",
+        message: "Use relPath or folderPath, not both",
+        path: ["destination"],
+      });
+    }
+  });
+
+export type BrowserClipPayload = z.infer<typeof browserClipPayloadSchema>;
+export type BrowserClipBlock = z.infer<typeof readerBlockSchema>;
+
+export interface PreparedBrowserClip {
+  payload: BrowserClipPayload;
+  captureInput: CaptureInput;
+  provenance: BrowserClipProvenance;
+  preview: {
+    body: string;
+    digest: string;
+    source: CaptureSource;
+    destination: BrowserClipPayload["destination"];
+    tags: string[];
+  };
+}
+
+const normalizeUrl = (value: string): string => {
+  const url = new URL(value);
+  url.hash = "";
+  url.searchParams.sort();
+  return url.toString();
+};
+
+const normalizePublishedAt = (value: string | null): string | null => {
+  if (value === null || /^\d{4}-\d{2}-\d{2}$/.test(value)) return value;
+  return new Date(value).toISOString();
+};
+
+const escapeInline = (value: string): string =>
+  value
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replace(/([\\*_[\]~`])/g, "\\$1");
+
+const normalizeLink = (value: string): string =>
+  normalizeUrl(value).replaceAll("(", "%28").replaceAll(")", "%29");
+
+const renderInline = (content: z.infer<typeof inlineContentSchema>): string =>
+  content
+    .map((node) =>
+      node.type === "text"
+        ? escapeInline(node.text)
+        : `[${escapeInline(node.text)}](${normalizeLink(node.href)})`
+    )
+    .join("");
+
+const codeFence = (text: string): string => {
+  const longest = Math.max(
+    0,
+    ...[...text.matchAll(/`+/g)].map((match) => match[0].length)
+  );
+  return "`".repeat(Math.max(3, longest + 1));
+};
+
+export const renderBrowserClipReader = (
+  blocks: readonly BrowserClipBlock[]
+): string =>
+  canonicalize(
+    blocks
+      .map((block) => {
+        switch (block.type) {
+          case "paragraph":
+            return renderInline(block.content);
+          case "heading":
+            return `${"#".repeat(block.level)} ${renderInline(block.content)}`;
+          case "quote":
+            return renderInline(block.content)
+              .split("\n")
+              .map((line) => `> ${line}`)
+              .join("\n");
+          case "list":
+            return block.items
+              .map(
+                (item, index) =>
+                  `${block.ordered ? `${index + 1}.` : "-"} ${renderInline(item)}`
+              )
+              .join("\n");
+          case "code": {
+            const fence = codeFence(block.text);
+            return `${fence}${block.language ?? ""}\n${block.text}\n${fence}`;
+          }
+          case "horizontal_rule":
+            return "---";
+        }
+      })
+      .join("\n\n")
+  );
+
+const INLINE_MARKDOWN_LINK = /(?<!!)\[[^\]\r\n]*\]\(([^()\s]+)\)/gu;
+const REFERENCE_LINK = /\[[^\]\r\n]*\]\s*\[[^\]\r\n]*\]/u;
+const REFERENCE_DEFINITION = /^[ \t]{0,3}\[[^\]\r\n]+\]:/mu;
+
+const assertSafeEditedMarkdown = (markdown: string): void => {
+  if (markdown.includes("<")) {
+    throw new Error(
+      "Edited clip Markdown cannot contain HTML, comments, or autolinks."
+    );
+  }
+  if (markdown.includes("![")) {
+    throw new Error("Edited clip Markdown cannot contain images.");
+  }
+  if (REFERENCE_LINK.test(markdown) || REFERENCE_DEFINITION.test(markdown)) {
+    throw new Error("Edited clip Markdown cannot contain reference links.");
+  }
+  const unmatchedInlineSyntax = markdown.replace(
+    INLINE_MARKDOWN_LINK,
+    (_link, destination: string) => {
+      browserClipHttpUrlSchema.parse(destination);
+      return "";
+    }
+  );
+  if (unmatchedInlineSyntax.includes("](")) {
+    throw new Error(
+      "Edited clip Markdown links must use a simple absolute HTTP(S) destination."
+    );
+  }
+};
+
+const notePrefix = (note: string | null): string => {
+  const trimmed = note?.trim();
+  if (!trimmed) return "";
+  return `> **Clip note:** ${escapeInline(trimmed).replaceAll("\n", "\n> ")}\n\n`;
+};
+
+const dedupeWarnings = (
+  values: readonly BrowserClipWarningCode[]
+): BrowserClipWarningCode[] => [...new Set(values)].sort();
+
+export const prepareBrowserClip = (
+  input: unknown,
+  options: { now?: Date } = {}
+): PreparedBrowserClip => {
+  const payload = browserClipPayloadSchema.parse(input);
+  const capturedAt = (options.now ?? new Date()).toISOString();
+  const sourceUrl = normalizeUrl(payload.sourceUrl);
+  const canonicalUrl =
+    payload.canonicalUrl === null ? null : normalizeUrl(payload.canonicalUrl);
+  const extracted =
+    payload.mode === "selection"
+      ? payload.selection.exactText
+      : stableJson(payload.reader.blocks);
+  const edited =
+    payload.mode === "selection"
+      ? payload.selection.editedMarkdown
+      : payload.reader.editedMarkdown;
+  if (edited !== null) assertSafeEditedMarkdown(edited);
+  const uneditedBody =
+    payload.mode === "selection"
+      ? escapeInline(payload.selection.exactText)
+      : renderBrowserClipReader(payload.reader.blocks);
+  const body = canonicalize(
+    `${notePrefix(payload.note)}${edited ?? uneditedBody}`
+  );
+  const extractionHash = sha256(extracted);
+  const finalBodyHash = hashCaptureContent(body);
+  const clipIdentity = sha256(
+    stableJson({
+      canonicalUrl,
+      extractionHash,
+      finalBodyHash,
+      mode: payload.mode,
+      schemaVersion: payload.schemaVersion,
+      sourceUrl,
+    })
+  );
+  const warnings: BrowserClipWarningCode[] = [...payload.extraction.warnings];
+  if (payload.extraction.authenticated)
+    warnings.push("authenticated_visible_content");
+  if (canonicalUrl !== null && canonicalUrl !== sourceUrl) {
+    warnings.push("canonical_url_differs");
+  }
+  if (edited !== null) warnings.push("edited_content");
+  if (extracted.includes("\r")) warnings.push("line_endings_normalized");
+  if (extracted.normalize("NFC") !== extracted)
+    warnings.push("unicode_normalized");
+  const extractionWarnings = dedupeWarnings(warnings);
+  const publishedAt = normalizePublishedAt(payload.publishedAt);
+  const observedAt = new Date(payload.observedAt).toISOString();
+  const exactSelection =
+    payload.mode === "selection" ? payload.selection.exactText : null;
+  const previewSource = {
+    kind: "web",
+    title: payload.title,
+    url: sourceUrl,
+    author: payload.author,
+    observedAt,
+    canonicalUrl,
+    site: payload.site,
+    publishedAt,
+  };
+  const previewProvenance = {
+    schemaVersion: payload.schemaVersion,
+    mode: payload.mode,
+    sourceUrl,
+    canonicalUrl,
+    title: payload.title,
+    author: payload.author,
+    site: payload.site,
+    publishedAt,
+    observedAt,
+    extractionHash,
+    finalBodyHash,
+    clipIdentity,
+    exactSelection,
+    extractionWarnings,
+    browser: payload.browser,
+  };
+  const previewDigest = sha256(
+    stableJson({
+      body,
+      destination: payload.destination,
+      extraction: payload.extraction,
+      provenance: previewProvenance,
+      source: previewSource,
+      tags: payload.tags,
+    })
+  );
+  const provenance: BrowserClipProvenance = {
+    schemaVersion: payload.schemaVersion,
+    mode: payload.mode,
+    sourceUrl,
+    canonicalUrl,
+    title: payload.title,
+    author: payload.author,
+    site: payload.site,
+    publishedAt,
+    observedAt,
+    capturedAt,
+    extractionHash,
+    finalBodyHash,
+    clipIdentity,
+    previewDigest,
+    exactSelection,
+    extractionWarnings,
+    browser: payload.browser,
+  };
+  const source: CaptureSource = {
+    kind: "web",
+    title: payload.title,
+    url: sourceUrl,
+    author: payload.author ?? undefined,
+    observedAt: provenance.observedAt,
+    capturedAt,
+    canonicalUrl: canonicalUrl ?? undefined,
+    site: payload.site ?? undefined,
+    publishedAt: provenance.publishedAt ?? undefined,
+    browserClip: provenance,
+  };
+  const captureInput: CaptureInput = {
+    collection: payload.destination.collection,
+    content: body,
+    title: payload.title,
+    relPath: payload.destination.relPath ?? undefined,
+    folderPath: payload.destination.folderPath ?? undefined,
+    collisionPolicy: payload.destination.collisionPolicy,
+    tags: payload.tags,
+    source,
+  };
+  return {
+    payload,
+    captureInput,
+    provenance,
+    preview: {
+      body,
+      digest: previewDigest,
+      source,
+      destination: payload.destination,
+      tags: payload.tags,
+    },
+  };
+};

package/src/core/capture-write.ts

@@ -21,6 +21,11 @@ export async function writeCapturePlanFile(
   plan: CapturePlan,
   absPath: string
 ): Promise<void> {
+  if (plan.provenanceConflict) {
+    throw new Error(
+      "Existing capture has different provenance. Use create_with_suffix or a different destination."
+    );
+  }
   try {
     if (plan.overwrite) {
       await atomicWrite(absPath, plan.content);