@gmickel/gno 1.27.0 → 1.28.0

package/src/converters/adapters/jsonl/adapter.ts

@@ -0,0 +1,238 @@
+import type {
+  RecordAdapter,
+  RecordAdapterEvent,
+  RecordAdapterInput,
+  RecordAdapterRecord,
+  RecordMetadata,
+} from "../../types";
+
+import {
+  adapterLineByteLimit,
+  canonicalJson,
+  hashRecordValue,
+  safeInlineText,
+  safeMarkdownText,
+  scalarList,
+  scalarText,
+  sourceNamespace,
+} from "../shared/record-utils";
+import { readBoundedUtf8Lines } from "../shared/utf8-lines";
+import {
+  type JsonlFieldMapping,
+  parseJsonlFieldMapping,
+  resolveJsonlField,
+} from "./config";
+
+const ADAPTER_ID = "adapter/jsonl";
+const ADAPTER_VERSION = "1.0.0";
+const DEFAULT_ID_SELECTORS = ["/id", "/externalId", "/external_id"];
+const DEFAULT_BODY_SELECTORS = ["/text", "/content", "/body"];
+const DEFAULT_TITLE_SELECTORS = ["/title", "/name"];
+const DEFAULT_AUTHOR_SELECTORS = ["/author", "/creator"];
+
+const failure = (
+  code: "MALFORMED_RECORD" | "MISSING_ID" | "RECORD_TOO_LARGE",
+  lineNumber: number,
+  retryable = false
+): RecordAdapterEvent => ({
+  type: "failure",
+  failure: {
+    code,
+    message: "JSONL record could not be converted.",
+    retryable,
+    sourceLocator: `line:${lineNumber}`,
+  },
+});
+
+const requireScalar = (
+  value: unknown
+): { ok: true; value?: string } | { ok: false } => {
+  if (value === undefined || value === null) return { ok: true };
+  const scalar = scalarText(value);
+  return scalar ? { ok: true, value: scalar } : { ok: false };
+};
+
+const markdownBody = (value: unknown): string => {
+  const scalar = scalarText(value);
+  if (scalar !== undefined) return safeMarkdownText(scalar);
+  return safeMarkdownText(canonicalJson(value));
+};
+
+const buildMetadata = (
+  source: Record<string, unknown>,
+  mapping: JsonlFieldMapping
+): { ok: true; metadata: RecordMetadata } | { ok: false } => {
+  const author = requireScalar(resolveJsonlField(source, mapping.author));
+  const sessionId = requireScalar(resolveJsonlField(source, mapping.sessionId));
+  const threadId = requireScalar(resolveJsonlField(source, mapping.threadId));
+  if (!(author.ok && sessionId.ok && threadId.ok)) return { ok: false };
+
+  const participantsValue = resolveJsonlField(source, mapping.participants);
+  const categoriesValue = resolveJsonlField(source, mapping.categories);
+  const participants = scalarList(participantsValue);
+  const categories = scalarList(categoriesValue);
+  if (
+    (participantsValue !== undefined && !participants) ||
+    (categoriesValue !== undefined && !categories)
+  ) {
+    return { ok: false };
+  }
+
+  const dateFields: Record<string, string> = {};
+  for (const [name, selector] of Object.entries(mapping.dateFields ?? {})) {
+    const raw = resolveJsonlField(source, selector);
+    const value = requireScalar(raw);
+    if (!value.ok) return { ok: false };
+    if (value.value) dateFields[name] = value.value;
+  }
+
+  return {
+    ok: true,
+    metadata: {
+      author: author.value,
+      participants,
+      categories,
+      dateFields: Object.keys(dateFields).length > 0 ? dateFields : undefined,
+      sessionId: sessionId.value,
+      threadId: threadId.value,
+    },
+  };
+};
+
+const recordFromLine = (
+  source: Record<string, unknown>,
+  lineNumber: number,
+  input: RecordAdapterInput,
+  mapping: JsonlFieldMapping,
+  allowConventionalId: boolean
+): RecordAdapterEvent => {
+  const canonicalSource = canonicalJson(source);
+  const container = sourceNamespace(input);
+  const configuredId = resolveJsonlField(
+    source,
+    mapping.id ?? (allowConventionalId ? DEFAULT_ID_SELECTORS : undefined)
+  );
+  const id = requireScalar(configuredId);
+  if (!id.ok) return failure("MALFORMED_RECORD", lineNumber);
+  if (mapping.id && !id.value) return failure("MISSING_ID", lineNumber);
+
+  const configuredBody = resolveJsonlField(
+    source,
+    mapping.body ?? DEFAULT_BODY_SELECTORS
+  );
+  if (mapping.body && configuredBody === undefined) {
+    return failure("MALFORMED_RECORD", lineNumber);
+  }
+  const body = configuredBody ?? source;
+  const titleValue = requireScalar(
+    resolveJsonlField(source, mapping.title ?? DEFAULT_TITLE_SELECTORS)
+  );
+  if (!titleValue.ok) return failure("MALFORMED_RECORD", lineNumber);
+  const authorMapping = mapping.author
+    ? mapping
+    : { ...mapping, author: DEFAULT_AUTHOR_SELECTORS };
+  const metadata = buildMetadata(source, authorMapping);
+  if (!metadata.ok) return failure("MALFORMED_RECORD", lineNumber);
+
+  const identity = id.value
+    ? `id:${hashRecordValue("gno-jsonl-external-id-v1", id.value)}`
+    : `content:${hashRecordValue("gno-jsonl-content-id-v1", canonicalSource)}`;
+  const title = titleValue.value ?? `JSONL record ${lineNumber}`;
+  const record: RecordAdapterRecord = {
+    stableId: `jsonl:${container}:${identity}`,
+    sourceLocator: `line:${lineNumber}`,
+    sourceHash: hashRecordValue("gno-jsonl-source-v1", canonicalSource),
+    title,
+    markdown: `# ${safeInlineText(title)}\n\n${markdownBody(body)}`,
+    metadata: metadata.metadata,
+    anchors: [{ kind: "line", value: String(lineNumber) }],
+  };
+  return { type: "record", record };
+};
+
+export const createJsonlAdapter = (
+  fieldMapping?: JsonlFieldMapping
+): RecordAdapter => {
+  const mapping = parseJsonlFieldMapping(fieldMapping);
+  const allowConventionalId = fieldMapping === undefined;
+  return {
+    id: ADAPTER_ID,
+    version: ADAPTER_VERSION,
+    configurationFingerprint: hashRecordValue(
+      "gno-jsonl-config-v1",
+      canonicalJson(mapping)
+    ),
+    canHandle: (mime, ext) =>
+      mime === "application/x-ndjson" ||
+      mime === "application/jsonl" ||
+      ext === ".jsonl" ||
+      ext === ".ndjson",
+    records: async function* (
+      input: RecordAdapterInput
+    ): AsyncGenerator<RecordAdapterEvent> {
+      let hadFailure = false;
+      try {
+        const lines = readBoundedUtf8Lines(
+          input.open(),
+          adapterLineByteLimit(input)
+        );
+        for await (const line of lines) {
+          if (!line.ok) {
+            hadFailure = true;
+            yield failure(
+              line.reason === "line_too_large"
+                ? "RECORD_TOO_LARGE"
+                : "MALFORMED_RECORD",
+              line.lineNumber,
+              !line.terminated
+            );
+            continue;
+          }
+          if (line.text.trim() === "") continue;
+          let parsed: unknown;
+          try {
+            parsed = JSON.parse(line.text);
+          } catch {
+            hadFailure = true;
+            yield failure(
+              "MALFORMED_RECORD",
+              line.lineNumber,
+              !line.terminated
+            );
+            continue;
+          }
+          if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+            hadFailure = true;
+            yield failure("MALFORMED_RECORD", line.lineNumber);
+            continue;
+          }
+          const event = recordFromLine(
+            parsed as Record<string, unknown>,
+            line.lineNumber,
+            input,
+            mapping,
+            allowConventionalId
+          );
+          hadFailure ||= event.type === "failure";
+          yield event;
+        }
+      } catch {
+        hadFailure = true;
+        yield {
+          type: "failure",
+          failure: {
+            code: "ADAPTER_FAILURE",
+            message: "JSONL source could not be read completely.",
+            retryable: true,
+          },
+        };
+      }
+      yield {
+        type: "snapshot",
+        state: hadFailure ? "partial" : "complete",
+      };
+    },
+  };
+};
+
+export const jsonlAdapter = createJsonlAdapter();

package/src/converters/adapters/jsonl/config.ts

@@ -0,0 +1,105 @@
+import { z } from "zod";
+
+const FORBIDDEN_POINTER_SEGMENTS = new Set([
+  "__proto__",
+  "constructor",
+  "prototype",
+]);
+const POINTER_PATTERN = /^(?:\/(?:[^~/]|~[01])*)*$/;
+const FIELD_NAME_PATTERN = /^[a-z][a-z0-9_-]{0,63}$/;
+
+const JsonPointerSchema = z
+  .string()
+  .max(512)
+  .refine((value) => POINTER_PATTERN.test(value), "Invalid JSON Pointer")
+  .refine(
+    (value) => value.split("/").length <= 33,
+    "JSON Pointer exceeds 32 segments"
+  )
+  .refine(
+    (value) =>
+      value
+        .split("/")
+        .slice(1)
+        .map((segment) => segment.replaceAll("~1", "/").replaceAll("~0", "~"))
+        .every((segment) => !FORBIDDEN_POINTER_SEGMENTS.has(segment)),
+    "JSON Pointer contains a forbidden property"
+  );
+
+const FieldSelectorSchema = z.union([
+  JsonPointerSchema,
+  z.array(JsonPointerSchema).min(1).max(8),
+]);
+
+export const JsonlFieldMappingSchema = z
+  .object({
+    id: FieldSelectorSchema.optional(),
+    body: FieldSelectorSchema.optional(),
+    title: FieldSelectorSchema.optional(),
+    author: FieldSelectorSchema.optional(),
+    participants: FieldSelectorSchema.optional(),
+    categories: FieldSelectorSchema.optional(),
+    sessionId: FieldSelectorSchema.optional(),
+    threadId: FieldSelectorSchema.optional(),
+    dateFields: z
+      .record(z.string().regex(FIELD_NAME_PATTERN), FieldSelectorSchema)
+      .superRefine((fields, context) => {
+        if (Object.keys(fields).length > 16) {
+          context.addIssue({
+            code: "custom",
+            message: "JSONL date-field mapping exceeds 16 entries",
+          });
+        }
+      })
+      .optional(),
+  })
+  .strict();
+
+export type JsonlFieldSelector = z.infer<typeof FieldSelectorSchema>;
+export type JsonlFieldMapping = z.infer<typeof JsonlFieldMappingSchema>;
+
+export const parseJsonlFieldMapping = (
+  mapping: JsonlFieldMapping | undefined
+): JsonlFieldMapping => JsonlFieldMappingSchema.parse(mapping ?? {});
+
+const pointerSegments = (pointer: string): string[] =>
+  pointer
+    .split("/")
+    .slice(1)
+    .map((segment) => segment.replaceAll("~1", "/").replaceAll("~0", "~"));
+
+export const resolveJsonPointer = (
+  value: unknown,
+  pointer: string
+): unknown => {
+  let current = value;
+  for (const segment of pointerSegments(pointer)) {
+    if (Array.isArray(current)) {
+      if (!/^(?:0|[1-9]\d*)$/.test(segment)) return undefined;
+      current = current[Number(segment)];
+      continue;
+    }
+    if (
+      !current ||
+      typeof current !== "object" ||
+      !Object.hasOwn(current, segment)
+    ) {
+      return undefined;
+    }
+    current = (current as Record<string, unknown>)[segment];
+  }
+  return current;
+};
+
+export const resolveJsonlField = (
+  value: unknown,
+  selector: JsonlFieldSelector | undefined
+): unknown => {
+  if (selector === undefined) return undefined;
+  const pointers = Array.isArray(selector) ? selector : [selector];
+  for (const pointer of pointers) {
+    const resolved = resolveJsonPointer(value, pointer);
+    if (resolved !== undefined && resolved !== null) return resolved;
+  }
+  return undefined;
+};

package/src/converters/adapters/shared/html-text.ts

@@ -0,0 +1,165 @@
+const NAMED_ENTITIES = new Map<string, string>([
+  ["amp", "&"],
+  ["apos", "'"],
+  ["gt", ">"],
+  ["lt", "<"],
+  ["quot", '"'],
+]);
+const MAX_ENTITY_CHARS = 16;
+
+const isAsciiWhitespace = (value: string): boolean =>
+  value === " " ||
+  value === "\t" ||
+  value === "\n" ||
+  value === "\r" ||
+  value === "\f";
+
+const isTagNameCharacter = (value: string): boolean => {
+  const code = value.charCodeAt(0);
+  return (
+    (code >= 48 && code <= 57) ||
+    (code >= 65 && code <= 90) ||
+    (code >= 97 && code <= 122) ||
+    value === "-"
+  );
+};
+
+const isHexDigit = (value: string): boolean => {
+  const code = value.charCodeAt(0);
+  return (
+    (code >= 48 && code <= 57) ||
+    (code >= 65 && code <= 70) ||
+    (code >= 97 && code <= 102)
+  );
+};
+
+const hasOnlyDigits = (value: string, hexadecimal: boolean): boolean => {
+  for (let index = 0; index < value.length; index += 1) {
+    const digit = value[index] ?? "";
+    if (hexadecimal ? !isHexDigit(digit) : digit < "0" || digit > "9")
+      return false;
+  }
+  return value.length > 0;
+};
+
+/** Decode recognized entities from the original input exactly once. */
+export const decodeHtmlEntitiesOnce = (value: string): string => {
+  let output = "";
+  let cursor = 0;
+  while (cursor < value.length) {
+    if (value[cursor] !== "&") {
+      output += value[cursor];
+      cursor += 1;
+      continue;
+    }
+    const entityEnd = Math.min(value.length, cursor + MAX_ENTITY_CHARS + 2);
+    const semicolon = value.slice(cursor + 1, entityEnd).indexOf(";");
+    const absoluteSemicolon = semicolon < 0 ? -1 : cursor + semicolon + 1;
+    if (
+      absoluteSemicolon < 0 ||
+      absoluteSemicolon - cursor - 1 <= 0 ||
+      absoluteSemicolon - cursor - 1 > MAX_ENTITY_CHARS
+    ) {
+      output += "&";
+      cursor += 1;
+      continue;
+    }
+    const entity = value.slice(cursor + 1, absoluteSemicolon);
+    const named = NAMED_ENTITIES.get(entity.toLowerCase());
+    let decoded = named;
+    if (!decoded && entity.startsWith("#")) {
+      const hexadecimal =
+        entity.length > 2 && (entity[1] === "x" || entity[1] === "X");
+      const digits = entity.slice(hexadecimal ? 2 : 1);
+      const base = hexadecimal ? 16 : 10;
+      const codePoint = hasOnlyDigits(digits, hexadecimal)
+        ? Number.parseInt(digits, base)
+        : Number.NaN;
+      if (
+        Number.isSafeInteger(codePoint) &&
+        codePoint > 0 &&
+        codePoint <= 0x10ffff &&
+        !(codePoint >= 0xd800 && codePoint <= 0xdfff)
+      ) {
+        decoded = String.fromCodePoint(codePoint);
+      }
+    }
+    if (decoded === undefined) {
+      output += value.slice(cursor, absoluteSemicolon + 1);
+    } else {
+      output += decoded;
+    }
+    cursor = absoluteSemicolon + 1;
+  }
+  return output;
+};
+
+interface HtmlTag {
+  closing: boolean;
+  end: number;
+  name: string;
+}
+
+const readTag = (source: string, start: number): HtmlTag | undefined => {
+  const end = source.indexOf(">", start + 1);
+  if (end < 0) return undefined;
+  let cursor = start + 1;
+  while (cursor < end && isAsciiWhitespace(source[cursor] ?? "")) cursor += 1;
+  const closing = source[cursor] === "/";
+  if (closing) {
+    cursor += 1;
+    while (cursor < end && isAsciiWhitespace(source[cursor] ?? "")) cursor += 1;
+  }
+  const nameStart = cursor;
+  while (cursor < end && isTagNameCharacter(source[cursor] ?? "")) cursor += 1;
+  return {
+    closing,
+    end,
+    name: source.slice(nameStart, cursor).toLowerCase(),
+  };
+};
+
+/**
+ * Reduce a bounded HTML fragment to inert text with a linear scanner.
+ * Script/style contents and comments are discarded; entities decode once.
+ */
+export const htmlFragmentToText = (source: string): string => {
+  let output = "";
+  let cursor = 0;
+  let blockedName: "script" | "style" | undefined;
+  let blockedDepth = 0;
+  while (cursor < source.length) {
+    if (source[cursor] !== "<") {
+      if (!blockedName) output += source[cursor];
+      cursor += 1;
+      continue;
+    }
+    if (source.startsWith("<!--", cursor)) {
+      const commentEnd = source.indexOf("-->", cursor + 4);
+      if (commentEnd < 0) break;
+      cursor = commentEnd + 3;
+      continue;
+    }
+    const tag = readTag(source, cursor);
+    if (!tag) {
+      if (!blockedName) output += source.slice(cursor);
+      break;
+    }
+    cursor = tag.end + 1;
+    if (blockedName) {
+      if (tag.name === blockedName) {
+        if (tag.closing) blockedDepth -= 1;
+        else blockedDepth += 1;
+        if (blockedDepth === 0) blockedName = undefined;
+      }
+      continue;
+    }
+    if (!tag.closing && (tag.name === "script" || tag.name === "style")) {
+      blockedName = tag.name;
+      blockedDepth = 1;
+      continue;
+    }
+    if (!tag.closing && tag.name === "br") output += "\n";
+  }
+  return decodeHtmlEntitiesOnce(output);
+};

package/src/converters/adapters/shared/record-utils.ts

@@ -0,0 +1,79 @@
+import type { RecordAdapterInput } from "../../types";
+
+const CONTROL_PATTERN = new RegExp(
+  `[${String.fromCharCode(0)}-${String.fromCharCode(8)}${String.fromCharCode(11)}${String.fromCharCode(12)}${String.fromCharCode(14)}-${String.fromCharCode(31)}${String.fromCharCode(127)}]`,
+  "g"
+);
+
+export const hashRecordValue = (domain: string, value: string): string => {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(`${domain}\0${value}`);
+  return hasher.digest("hex");
+};
+
+export const canonicalJson = (value: unknown): string => {
+  const normalize = (candidate: unknown): unknown => {
+    if (Array.isArray(candidate)) return candidate.map(normalize);
+    if (candidate && typeof candidate === "object") {
+      const record = candidate as Record<string, unknown>;
+      return Object.fromEntries(
+        Object.keys(record)
+          .sort()
+          .map((key) => [key, normalize(record[key])])
+      );
+    }
+    return candidate;
+  };
+  return JSON.stringify(normalize(value));
+};
+
+export const safeMarkdownText = (value: string): string =>
+  value
+    .normalize("NFC")
+    .replace(CONTROL_PATTERN, "")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replace(/([\\`*_[\]#!])/g, "\\$1");
+
+export const safeInlineText = (value: string): string =>
+  safeMarkdownText(value).replaceAll(/\s+/g, " ").trim();
+
+export const sourceNamespace = (input: RecordAdapterInput): string =>
+  hashRecordValue(
+    "gno-record-container-v1",
+    canonicalJson({
+      collection: input.collection.normalize("NFC"),
+      relativePath: input.relativePath.replaceAll("\\", "/").normalize("NFC"),
+    })
+  );
+
+export const adapterLineByteLimit = (input: RecordAdapterInput): number => {
+  const characterBudget =
+    input.limits.maxRecordChars + input.limits.maxMetadataChars;
+  const utf8Budget = Math.min(
+    Number.MAX_SAFE_INTEGER,
+    Math.max(1, characterBudget) * 4
+  );
+  return Math.max(1, Math.min(input.limits.maxSourceBytes, utf8Budget));
+};
+
+export const scalarText = (value: unknown): string | undefined => {
+  if (
+    typeof value !== "string" &&
+    typeof value !== "number" &&
+    typeof value !== "boolean"
+  ) {
+    return undefined;
+  }
+  const text = String(value).normalize("NFC").trim();
+  return text || undefined;
+};
+
+export const scalarList = (value: unknown): string[] | undefined => {
+  if (value === undefined || value === null) return undefined;
+  const candidates = Array.isArray(value) ? value : [value];
+  const result = candidates
+    .map(scalarText)
+    .filter((item): item is string => Boolean(item));
+  return result.length > 0 ? result : undefined;
+};