@hraness/wordcell 0.22.4 → 0.23.0

package/src/authoring-model.ts

@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
 import { posix, relative, resolve, sep } from "node:path";
 import { Document, isMap, isScalar, isSeq, parseDocument, type YAMLMap, type YAMLSeq } from "yaml";
 import { type NoteLockOptions } from "./note-lock.js";
-import { isCanonicalNoteId, isCanonicalRelationPredicate } from "./graph.js";
+import { isCanonicalNoteId, isCanonicalRelationPredicate, isMetadataNumber } from "./graph.js";
 import { parseDocumentId, parseQualifiedDocumentUri } from "./portfolio-identity.js";
 
 export const MAX_NOTE_BYTES = 16 * 1024 * 1024;
@@ -721,12 +721,188 @@ export function renderUpdatedNoteBody(
   return `${header}${parts.newline}${parts.newline}${normalized}`;
 }
 
-export function renderCreatedNote(input: CreateNoteInput, documentId: string): string {
+/**
+ * Extra top-level frontmatter values that internal writers, such as the
+ * supermemory importer, own. Public authoring never passes them.
+ */
+export type FrontmatterScalar = string | number | boolean;
+export type FrontmatterFieldValue =
+  | FrontmatterScalar
+  | readonly string[]
+  | Readonly<Record<string, FrontmatterScalar>>;
+export type FrontmatterFields = Readonly<Record<string, FrontmatterFieldValue>>;
+/** Field updates; `null` deletes the key. `title` is allowed here. */
+export type FrontmatterFieldUpdates = Readonly<Record<string, FrontmatterFieldValue | null>>;
+
+export const MAX_FRONTMATTER_FIELDS = 64;
+export const MAX_FRONTMATTER_LIST_ITEMS = 256;
+const FRONTMATTER_FIELD_KEY = /^[a-z][a-z0-9_]{0,63}$/u;
+const FRONTMATTER_MAP_KEY = /^[A-Za-z0-9_.:-]{1,64}$/u;
+const RESERVED_FRONTMATTER_FIELDS: ReadonlySet<string> = new Set([
+  "document_id", "type", "title", "tags", "relations",
+]);
+
+function validateFieldString(value: unknown, label: string): string {
+  if (typeof value !== "string") throw new TypeError(`frontmatter field ${label} must be a string`);
+  if (
+    Buffer.from(value, "utf8").toString("utf8") !== value
+    || /[\u0000-\u001f\u007f]/u.test(value)
+  ) {
+    throw new TypeError(`frontmatter field ${label} must be a single line of well-formed text`);
+  }
+  return value;
+}
+
+function validateFieldScalar(value: unknown, label: string): FrontmatterScalar {
+  if (typeof value === "boolean") return value;
+  if (typeof value === "number") {
+    if (!isMetadataNumber(value)) {
+      throw new TypeError(`frontmatter field ${label} must be a finite number and a safe integer when integral`);
+    }
+    return value;
+  }
+  return validateFieldString(value, label);
+}
+
+function validateFieldValue(value: unknown, key: string): FrontmatterFieldValue {
+  if (Array.isArray(value)) {
+    if (value.length > MAX_FRONTMATTER_LIST_ITEMS) {
+      throw new TypeError(`frontmatter field ${key} has too many items`);
+    }
+    return value.map((item, index) => validateFieldString(item, `${key}[${index}]`));
+  }
+  if (typeof value === "object" && value !== null) {
+    if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) {
+      throw new TypeError(`frontmatter field ${key} must be a plain map`);
+    }
+    const entries = Object.entries(value);
+    if (entries.length > MAX_FRONTMATTER_FIELDS) {
+      throw new TypeError(`frontmatter field ${key} has too many keys`);
+    }
+    const result: Record<string, FrontmatterScalar> = {};
+    for (const [nested, item] of entries) {
+      if (!FRONTMATTER_MAP_KEY.test(nested) || nested === "__proto__") {
+        throw new TypeError(`frontmatter field ${key} has an invalid key: ${JSON.stringify(nested)}`);
+      }
+      result[nested] = validateFieldScalar(item, `${key}.${nested}`);
+    }
+    return result;
+  }
+  return validateFieldScalar(value, key);
+}
+
+function validateFieldKey(key: string, allowTitle: boolean): void {
+  if (!FRONTMATTER_FIELD_KEY.test(key)) {
+    throw new TypeError(`not a valid frontmatter field name: ${JSON.stringify(key)}`);
+  }
+  if (RESERVED_FRONTMATTER_FIELDS.has(key) && !(allowTitle && key === "title")) {
+    throw new TypeError(`frontmatter field ${key} is reserved`);
+  }
+}
+
+/** Validate importer-owned fields in their given key order. */
+export function validateFrontmatterFields(fields: FrontmatterFields | undefined): FrontmatterFields {
+  if (fields === undefined) return {};
+  const entries = Object.entries(fields);
+  if (entries.length > MAX_FRONTMATTER_FIELDS) throw new TypeError("too many frontmatter fields");
+  const result: Record<string, FrontmatterFieldValue> = {};
+  for (const [key, value] of entries) {
+    validateFieldKey(key, false);
+    result[key] = validateFieldValue(value, key);
+  }
+  return result;
+}
+
+/** Validate field updates; `title` must pass `validateTitle` and `null` deletes. */
+export function validateFrontmatterFieldUpdates(
+  fields: FrontmatterFieldUpdates | undefined,
+): FrontmatterFieldUpdates {
+  if (fields === undefined) return {};
+  const entries = Object.entries(fields);
+  if (entries.length > MAX_FRONTMATTER_FIELDS) throw new TypeError("too many frontmatter fields");
+  const result: Record<string, FrontmatterFieldValue | null> = {};
+  for (const [key, value] of entries) {
+    validateFieldKey(key, true);
+    if (key === "title") {
+      if (typeof value !== "string") throw new TypeError("frontmatter field title must be a string");
+      result[key] = validateTitle(value);
+    } else {
+      result[key] = value === null ? null : validateFieldValue(value, key);
+    }
+  }
+  return result;
+}
+
+function canonicalJson(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  if (typeof value === "object" && value !== null) {
+    const keys = Object.keys(value).sort();
+    return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson((value as Record<string, unknown>)[key])}`).join(",")}}`;
+  }
+  return JSON.stringify(value) ?? "undefined";
+}
+
+/** The plain value of one top-level key, or `undefined` when absent. */
+export function topLevelValue(parts: FrontmatterParts, key: string): unknown {
+  const contents = parts.document.contents;
+  if (!isMap(contents) || !contents.has(key)) return undefined;
+  const node: unknown = contents.get(key, true);
+  if (typeof node === "object" && node !== null && "toJSON" in node && typeof node.toJSON === "function") {
+    return (node as { toJSON: () => unknown }).toJSON();
+  }
+  return node;
+}
+
+export function sameFrontmatterValue(present: unknown, requested: unknown): boolean {
+  return canonicalJson(present) === canonicalJson(requested);
+}
+
+/**
+ * Replace the body and set or delete top-level fields in one rendered write.
+ * With no fields this is exactly `renderUpdatedNoteBody`.
+ */
+export function renderUpdatedNoteBodyAndFields(
+  snapshot: Pick<NoteSnapshot, "content" | "relativePath">,
+  parts: FrontmatterParts,
+  body: string,
+  fields: FrontmatterFieldUpdates | undefined,
+): string {
+  const updates = validateFrontmatterFieldUpdates(fields);
+  const bodyOnly = renderUpdatedNoteBody(snapshot, parts, body);
+  if (Object.keys(updates).length === 0) return bodyOnly;
+  if (!parts.hadFrontmatter || !isMap(parts.document.contents)) {
+    throw new TypeError("frontmatter fields require a note with a frontmatter map");
+  }
+  const contents = parts.document.contents;
+  for (const [key, value] of Object.entries(updates)) {
+    if (value === null) {
+      contents.delete(key);
+    } else if (!sameFrontmatterValue(topLevelValue(parts, key), value)) {
+      contents.set(key, parts.document.createNode(value));
+    }
+  }
+  const content = renderFrontmatter({
+    ...parts,
+    bodySuffix: `${parts.newline}${parts.newline}${normalizedRequestedBody(body)}`,
+  });
+  if (Buffer.byteLength(content, "utf8") > MAX_NOTE_BYTES) {
+    throw new RangeError("the note is too large for bounded authoring");
+  }
+  return content;
+}
+
+export function renderCreatedNote(
+  input: CreateNoteInput,
+  documentId: string,
+  fields?: FrontmatterFields,
+): string {
   const title = validateTitle(input.title);
   const type = validateType(input.type);
   const tags = validateTags(input.tags);
+  const extra = validateFrontmatterFields(fields);
   const metadata: Record<string, unknown> = { document_id: documentId, type, title };
   if (tags.length > 0) metadata["tags"] = tags;
+  for (const [key, value] of Object.entries(extra)) metadata[key] = value;
   const document = new Document(metadata, { schema: "core" });
   const body = normalizedRequestedBody(input.body ?? `# ${title}\n`);
   return `---\n${document.toString({ lineWidth: 0 })}---\n\n${body}`;
@@ -786,8 +962,10 @@ export function assertCompatibleCreate(
   snapshot: NoteSnapshot,
   input: CreateNoteInput,
   requestedDocumentId: string | undefined,
+  fields?: FrontmatterFields,
 ): CompatibleCreate {
   const parts = frontmatter(snapshot.content, snapshot.relativePath);
+  const requestedFields = validateFrontmatterFields(fields);
   const requestedType = validateType(input.type);
   const requestedTitle = validateTitle(input.title);
   if (topLevelScalar(parts, "type") !== requestedType) {
@@ -810,6 +988,11 @@ export function assertCompatibleCreate(
   ) {
     throw new NoteAlreadyExistsError(snapshot.relativePath, "body differs");
   }
+  for (const [key, value] of Object.entries(requestedFields)) {
+    if (!sameFrontmatterValue(topLevelValue(parts, key), value)) {
+      throw new NoteAlreadyExistsError(snapshot.relativePath, `${key} differs`);
+    }
+  }
   const existingId = existingDocumentId(parts);
   if (
     requestedDocumentId !== undefined

package/src/authoring-program.ts

@@ -4,12 +4,12 @@ import {
   MAX_NOTE_BYTES, NoteRevisionConflictError, addRelationToParts, assertCompatibleCreate,
   assertExpected, canonicalNoteId, canonicalRelationTarget, checkedExpectedRevision,
   existingDocumentId, frontmatter, isErrno, normalizeRelationPredicate, noteResult, pathFor, relationsFromParts,
-  removeRelationFromParts, renderCreatedNote, renderFrontmatter, renderUpdatedNoteBody, requireRevision, revisionFor,
-  sameQuarantinedSnapshot, sameSnapshot, withRecoveryPath,
+  removeRelationFromParts, renderCreatedNote, renderFrontmatter, renderUpdatedNoteBodyAndFields, requireRevision, revisionFor,
+  sameQuarantinedSnapshot, sameSnapshot, validateFrontmatterFieldUpdates, validateFrontmatterFields, withRecoveryPath,
 } from "./authoring-model.js";
 import type {
   AuthoringDependencies, AuthoringInstallContext, AuthoringOptions, CreateNoteInput,
-  DirectoryIdentity, NoteAuthoringResult, NoteRevision, NoteSnapshot, RecoveryLocation, UpdateNoteBodyOptions, Vault,
+  DirectoryIdentity, FrontmatterFieldUpdates, FrontmatterFields, NoteAuthoringResult, NoteRevision, NoteSnapshot, RecoveryLocation, UpdateNoteBodyOptions, Vault,
 } from "./authoring-model.js";
 import type { NoteLock } from "./note-lock.js";
 import { parseDocumentId } from "./portfolio-identity.js";
@@ -211,12 +211,18 @@ function installNote(
   });
 }
 
+/**
+ * `fields` is internal: extra top-level frontmatter keys owned by an importer.
+ * The public `createNote` never passes it.
+ */
 export function createNoteProgram(
   platform: AuthoringPlatform, root: string, input: CreateNoteInput, options: AuthoringOptions,
+  fields?: FrontmatterFields,
 ): Effect.Effect<NoteAuthoringResult, AuthoringFailure> {
   return Effect.gen(function*() {
     const vault = yield* authoringNative(() => platform.resolveVault(root));
     const id = yield* authoringSync(() => canonicalNoteId(input.id));
+    const extra = yield* authoringSync(() => validateFrontmatterFields(fields));
     const requestedDocumentId = yield* authoringSync(() => input.documentId === undefined ? undefined : parseDocumentId(input.documentId));
     const expected = yield* authoringSync(() => checkedExpectedRevision(options));
     const dependencies = yield* authoringSync(() => platform.dependenciesFor(options.dependencies));
@@ -227,12 +233,12 @@ export function createNoteProgram(
         const existing = yield* authoringNative(() => platform.readOptionalSnapshot(vault, id));
         if (existing !== null) {
           yield* authoringSync(() => assertExpected(existing, expected));
-          const compatible = yield* authoringSync(() => assertCompatibleCreate(existing, input, requestedDocumentId));
+          const compatible = yield* authoringSync(() => assertCompatibleCreate(existing, input, requestedDocumentId, extra));
           return yield* authoringSync(() => noteResult(existing, compatible.relations, false, compatible.documentId));
         }
         if (expected !== undefined) return yield* fail(new NoteRevisionConflictError(`${id}.md`, expected, null));
         const documentId = yield* authoringSync(() => requestedDocumentId ?? parseDocumentId(dependencies.documentId()));
-        const content = yield* authoringSync(() => renderCreatedNote(input, documentId));
+        const content = yield* authoringSync(() => renderCreatedNote(input, documentId, extra));
         const revision = yield* installNote(platform, vault, id, content, null, lock, dependencies);
         return { changed: true, path: `${id}.md`, revision, relations: [], documentId };
       }),
@@ -277,12 +283,18 @@ export function editNoteRelationProgram(
   });
 }
 
+/**
+ * `fields` is internal: top-level frontmatter keys (and `title`) set in the
+ * same atomic write, where `null` deletes a key. The public `updateNoteBody`
+ * never passes it, so the authored frontmatter bytes stay untouched there.
+ */
 export function updateNoteBodyProgram(
   platform: AuthoringPlatform, root: string, idInput: string, body: string,
-  options: UpdateNoteBodyOptions,
+  options: UpdateNoteBodyOptions, fields?: FrontmatterFieldUpdates,
 ): Effect.Effect<NoteAuthoringResult, AuthoringFailure> {
   return Effect.gen(function*() {
     const id = yield* authoringSync(() => canonicalNoteId(idInput));
+    const updates = yield* authoringSync(() => validateFrontmatterFieldUpdates(fields));
     const expected = yield* authoringSync(() => {
       if (typeof options?.expectedRevision !== "string") {
         throw new TypeError("expectedRevision is required for a note body update");
@@ -302,7 +314,7 @@ export function updateNoteBodyProgram(
         if (identity.kind === "invalid") return yield* fail(new TypeError("the note has an invalid document_id"));
         const documentId = identity.kind === "valid" ? identity.documentId : undefined;
         const relations = yield* authoringSync(() => relationsFromParts(parts, source.relativePath));
-        const content = yield* authoringSync(() => renderUpdatedNoteBody(source, parts, body));
+        const content = yield* authoringSync(() => renderUpdatedNoteBodyAndFields(source, parts, body, updates));
         if (content === source.content) return yield* authoringSync(() => noteResult(source, relations, false, documentId));
         const revision = yield* installNote(platform, vault, id, content, source, lock, dependencies);
         return yield* authoringSync(() => noteResult({ relativePath: source.relativePath, revision }, relations, true, documentId));