@intentset/publisher 0.1.0

package/src/chunks.ts

@@ -0,0 +1,132 @@
+/**
+ * Retrieval chunks (Core §9): each published document split by level-two
+ * heading, then by paragraph where a section is longer than `maxChars`. Every
+ * chunk carries its document's publication block unchanged, so the access
+ * filters and provenance that held for the document hold for each piece of it
+ * and a retriever filters on the same fields before anything is assembled.
+ *
+ * Only published documents are chunked: chunks are made from the output of
+ * `publish`, never from canonical files.
+ */
+import { collectHeadings } from "@intentset/core";
+import type { PublicationMeta } from "./generate.ts";
+
+export interface ChunkOptions {
+  /** Upper bound on a chunk's text length; a single word longer than it is cut. Default 1200. */
+  maxChars?: number;
+}
+
+export interface Chunk {
+  /** `<document ID>/<n>`, n from 1 in document order. */
+  id: string;
+  document: string;
+  /** The document's level-one heading, or its ID when it has none. */
+  title: string;
+  /** The level-two heading the chunk falls under, or null before the first one. */
+  heading: string | null;
+  text: string;
+  publication: PublicationMeta;
+}
+
+/** What `toChunks` needs of a published document. */
+export interface ChunkSource {
+  id: string;
+  markset: string;
+  publication: PublicationMeta;
+}
+
+export function toChunks(documents: readonly ChunkSource[], options: ChunkOptions = {}): Chunk[] {
+  const maxChars = options.maxChars ?? 1200;
+  if (!Number.isInteger(maxChars) || maxChars < 1) throw new RangeError("maxChars must be a positive integer");
+  const chunks: Chunk[] = [];
+  for (const document of [...documents].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))) {
+    const lines = bodyLines(document.markset);
+    const headings = collectHeadings(lines, 0);
+    const title = headings.find((h) => h.depth === 1)?.text ?? document.id;
+    const skip = new Set(headings.filter((h) => h.depth <= 2).map((h) => h.line - 1));
+    const sections: { heading: string | null; lines: string[] }[] = [{ heading: null, lines: [] }];
+    const starts = new Map(headings.filter((h) => h.depth === 2).map((h) => [h.line - 1, h.text]));
+    lines.forEach((line, i) => {
+      const heading = starts.get(i);
+      if (heading !== undefined) sections.push({ heading, lines: [] });
+      else if (!skip.has(i)) sections[sections.length - 1].lines.push(line);
+    });
+    let n = 0;
+    for (const section of sections) {
+      for (const text of pack(blocks(section.lines), maxChars)) {
+        n++;
+        chunks.push({
+          id: `${document.id}/${n}`,
+          document: document.id,
+          title,
+          heading: section.heading,
+          text,
+          publication: document.publication,
+        });
+      }
+    }
+  }
+  return chunks;
+}
+
+/** The body of a generated document: after the frontmatter and the generated-file marker. */
+function bodyLines(markset: string): string[] {
+  const lines = markset.split("\n");
+  let start = 0;
+  if (lines[0] === "---") {
+    const close = lines.indexOf("---", 1);
+    start = close === -1 ? 0 : close + 1;
+  }
+  if (lines[start]?.startsWith("<!-- Generated by intentset publish")) start++;
+  return lines.slice(start);
+}
+
+/** Blank-line separated blocks, with a fenced code block kept whole. */
+function blocks(lines: readonly string[]): string[] {
+  const out: string[] = [];
+  let current: string[] = [];
+  let fence: string | null = null;
+  const flush = () => {
+    if (current.some((line) => line.trim() !== "")) out.push(current.join("\n").trim());
+    current = [];
+  };
+  for (const line of lines) {
+    const marker = /^ {0,3}(`{3,}|~{3,})/.exec(line)?.[1] ?? null;
+    if (fence === null && marker !== null) fence = marker;
+    else if (fence !== null && marker !== null && marker[0] === fence[0] && marker.length >= fence.length) fence = null;
+    if (fence === null && marker === null && line.trim() === "") flush();
+    else current.push(line);
+  }
+  flush();
+  return out;
+}
+
+/** Greedy packing of blocks into texts no longer than `max`, splitting a block only when it alone is longer. */
+function pack(items: readonly string[], max: number): string[] {
+  const out: string[] = [];
+  let current = "";
+  for (const item of items.flatMap((block) => split(block, max))) {
+    if (current === "") current = item;
+    else if (current.length + 2 + item.length <= max) current = `${current}\n\n${item}`;
+    else {
+      out.push(current);
+      current = item;
+    }
+  }
+  if (current !== "") out.push(current);
+  return out;
+}
+
+/** A block cut at whitespace into pieces no longer than `max`; a word longer than `max` is cut where it must be. */
+function split(block: string, max: number): string[] {
+  const out: string[] = [];
+  let rest = block;
+  while (rest.length > max) {
+    const cut = Math.max(rest.lastIndexOf(" ", max), rest.lastIndexOf("\n", max));
+    const at = cut > 0 ? cut : max;
+    out.push(rest.slice(0, at).trimEnd());
+    rest = rest.slice(at).trimStart();
+  }
+  if (rest !== "") out.push(rest);
+  return out;
+}

package/src/diagnostic.ts

@@ -0,0 +1,44 @@
+/**
+ * Publication diagnostics (ADR 0004): one shape, origin "publication",
+ * members in contract order, optional members present only when known.
+ *
+ * One rule governs every message built with this: a diagnostic may name an
+ * artifact the projection does not admit by its ID, never by its title or
+ * its path (Core §9, CLAUDE.md invariant 4). Callers pass `path: null` for
+ * such an artifact.
+ */
+import type { Diagnostic, Location } from "@intentset/core";
+
+export interface PublicationFields {
+  code: string;
+  severity?: "error" | "warning";
+  artifact: string | null;
+  path: string | null;
+  location?: Location;
+  field?: string;
+  message: string;
+  remediation: string;
+}
+
+export function publicationDiagnostic(fields: PublicationFields): Diagnostic {
+  return {
+    code: fields.code,
+    severity: fields.severity ?? "error",
+    origin: "publication",
+    artifact: fields.artifact,
+    path: fields.path,
+    ...(fields.location !== undefined ? { location: fields.location } : {}),
+    ...(fields.field !== undefined ? { field: fields.field } : {}),
+    message: fields.message,
+    remediation: fields.remediation,
+  };
+}
+
+export function compareStrings(a: string, b: string): number {
+  return a < b ? -1 : a > b ? 1 : 0;
+}
+
+/** Sorted, duplicates removed. */
+export function sortedUnique(values: Iterable<string>): string[] {
+  return [...new Set(values)].sort(compareStrings);
+}

package/src/generate.ts

@@ -0,0 +1,244 @@
+/**
+ * Generated Markset for one published knowledge artifact (Core §9,
+ * spec/publication.md).
+ *
+ * The document is the knowledge body, unchanged, under frontmatter that
+ * carries `markset: 0` and a top-level `intentset-publication` block: the
+ * generated profile `intentset/publication/0.1`, marked derived, with the
+ * provenance Core §9 requires. It is a top-level key, not `intentset:`, so
+ * nothing reads a generated document as a canonical record. A comment line
+ * after the frontmatter says the same to a person opening the file.
+ *
+ * Provenance names only what the projection may show. The knowledge's own
+ * path appears only in a projection that admits internal records; sources
+ * outside the projection are counted, not named; availability is the
+ * request's own values, so a document does not list the other editions,
+ * roles or releases its knowledge also serves.
+ *
+ * The result is checked as Markset before it is emitted: a Markset error is
+ * PUB003, and a `:::` directive outside Markset's vocabulary is PUB004
+ * (CLAUDE.md invariant 5: profiles never add a directive).
+ */
+import type { Artifact, Diagnostic, Graph, Visibility } from "@intentset/core";
+import { marksetCarrier } from "@intentset/markset-adapter";
+import { BLOCK_DIRECTIVE_NAMES, parseDocument } from "@markset-lang/parser";
+import { bodyLineOffset } from "./body.ts";
+import { publicationDiagnostic, sortedUnique } from "./diagnostic.ts";
+import type { Snapshot } from "./project.ts";
+import { type PublicationRequest, admits } from "./request.ts";
+import { type YamlValue, emitYaml } from "./yaml.ts";
+
+export const PUBLICATION_PROFILE = "intentset/publication/0.1";
+/** The top-level frontmatter key of a generated document. */
+export const PUBLICATION_KEY = "intentset-publication";
+
+export interface SourceStamp {
+  id: string;
+  /** The artifact's `revision`, or null when it declares none. */
+  revision: number | null;
+  sourceHash: string;
+}
+
+/** The `intentset-publication` block, in the order it is written. */
+export interface PublicationMeta {
+  profile: typeof PUBLICATION_PROFILE;
+  derived: true;
+  /** The projection the document was generated for; the document is as restricted as that projection. */
+  projection: Visibility;
+  /** The knowledge itself; `path` only in a projection that admits internal records. */
+  source: SourceStamp & { path?: string };
+  /** Explained artifacts the projection admits, sorted by ID. */
+  sources: SourceStamp[];
+  /** How many explained artifacts are outside the projection and so not named. */
+  withheldSources: number;
+  snapshot: Snapshot;
+  audience: string;
+  /** The request's values, and the flags the knowledge requires (every one of them named by the request). */
+  availability: { product: string; release: string; role: string; edition: string; flags: string[] };
+  reviewer: string;
+  reviewedAt: string;
+  publishedAt: string;
+}
+
+export interface GenerateOptions {
+  snapshot: Snapshot;
+  publishedAt: string;
+}
+
+export interface GeneratedDocument {
+  id: string;
+  /** `<ID>.md`. */
+  path: string;
+  markset: string;
+  publication: PublicationMeta;
+}
+
+/** The line after the frontmatter that marks a generated file. */
+export function markerLine(id: string): string {
+  return `<!-- Generated by intentset publish from ${id}; do not edit. -->`;
+}
+
+/** The provenance block for a knowledge artifact already selected by the projection and current in review. */
+export function publicationMeta(
+  graph: Graph,
+  artifact: Artifact,
+  request: PublicationRequest,
+  options: GenerateOptions,
+): PublicationMeta {
+  const { meta } = artifact;
+  if (meta.reviewedBy === undefined || meta.reviewedAt === undefined) {
+    throw new Error(`${meta.id} has no review; only knowledge the projection selected is generated`);
+  }
+  const source: PublicationMeta["source"] = {
+    id: meta.id,
+    revision: meta.revision ?? null,
+    sourceHash: artifact.sourceHash,
+  };
+  if (admits(request, "internal")) source.path = artifact.path;
+  const sources: SourceStamp[] = [];
+  let withheldSources = 0;
+  for (const id of sortedUnique(meta.links.explains ?? [])) {
+    const explained = graph.artifacts.get(id);
+    if (explained === undefined || !admits(request, explained.meta.visibility)) withheldSources++;
+    else sources.push({ id, revision: explained.meta.revision ?? null, sourceHash: explained.sourceHash });
+  }
+  return {
+    profile: PUBLICATION_PROFILE,
+    derived: true,
+    projection: request.visibility,
+    source,
+    sources,
+    withheldSources,
+    snapshot: { commit: options.snapshot.commit, graphHash: options.snapshot.graphHash },
+    audience: request.audience,
+    availability: {
+      product: request.product,
+      release: request.release,
+      role: request.role,
+      edition: request.edition,
+      flags: sortedUnique((meta.availability?.flags ?? []).filter((flag) => request.flags.includes(flag))),
+    },
+    reviewer: meta.reviewedBy,
+    reviewedAt: meta.reviewedAt,
+    publishedAt: options.publishedAt,
+  };
+}
+
+/** The document text, and the 1-based line the body's first line lands on. */
+export function marksetSource(publication: PublicationMeta, body: string): { source: string; bodyStartLine: number } {
+  const yaml = emitYaml({ markset: 0, [PUBLICATION_KEY]: publication as unknown as YamlValue });
+  const head = `---\n${yaml}---\n${markerLine(publication.source.id)}\n`;
+  const separated = body.startsWith("\n");
+  const headLines = head.split("\n").length - 1;
+  return {
+    source: separated ? head + body : `${head}\n${body}`,
+    bodyStartLine: headLines + (separated ? 1 : 2),
+  };
+}
+
+/** Generate and check one document. A document with any error is not returned. */
+export function generateDocument(
+  graph: Graph,
+  artifact: Artifact,
+  request: PublicationRequest,
+  options: GenerateOptions,
+): { document: GeneratedDocument | null; diagnostics: Diagnostic[] } {
+  const id = artifact.meta.id;
+  const path = `${id}.md`;
+  let publication: PublicationMeta;
+  let generated: { source: string; bodyStartLine: number };
+  try {
+    publication = publicationMeta(graph, artifact, request, options);
+    generated = marksetSource(publication, artifact.body);
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    return {
+      document: null,
+      diagnostics: [
+        publicationDiagnostic({
+          code: "PUB003",
+          artifact: id,
+          path: artifact.path,
+          message: `The generated document for ${id} could not be written as Markset: ${reason}.`,
+          remediation: "Correct the metadata value named in the message; provenance is written in plain YAML scalars.",
+        }),
+      ],
+    };
+  }
+  const diagnostics = checkGenerated(artifact, generated.source, generated.bodyStartLine);
+  if (diagnostics.length > 0) return { document: null, diagnostics };
+  return { document: { id, path, markset: generated.source, publication }, diagnostics };
+}
+
+/** PUB004 for a directive outside Markset's vocabulary, PUB003 for every other Markset error. */
+export function checkGenerated(artifact: Artifact, source: string, bodyStartLine: number): Diagnostic[] {
+  const id = artifact.meta.id;
+  const offset = bodyLineOffset(artifact);
+  const locate = (line: number | undefined) =>
+    line === undefined || offset === null || line < bodyStartLine
+      ? undefined
+      : { line: line - bodyStartLine + 1 + offset };
+  const lines = source.split("\n");
+  const parsed = parseDocument(source);
+  const diagnostics: Diagnostic[] = [];
+
+  const directiveLines = new Set<number>();
+  const visit = (node: unknown): void => {
+    if (typeof node !== "object" || node === null) return;
+    const n = node as {
+      type?: string;
+      name?: string | null;
+      position?: { start: { line: number } };
+      children?: unknown[];
+    };
+    if (n.type === "directive") {
+      const line = n.position?.start.line;
+      const name =
+        n.name ??
+        (line === undefined ? null : (/^\s*:{3,}\s*([A-Za-z][A-Za-z0-9_-]*)/.exec(lines[line - 1])?.[1] ?? null));
+      if (name !== null && !BLOCK_DIRECTIVE_NAMES.has(name)) {
+        if (line !== undefined) directiveLines.add(line);
+        diagnostics.push(
+          publicationDiagnostic({
+            code: "PUB004",
+            artifact: id,
+            path: artifact.path,
+            location: locate(line),
+            message: `${id} uses the directive :::${name}, which is not in Markset's vocabulary.`,
+            remediation: `Use one of Markset's constructs (${[...BLOCK_DIRECTIVE_NAMES].join(", ")}) or plain Markdown; an Intentset profile never adds a directive (Core §9).`,
+          }),
+        );
+      }
+    }
+    for (const child of n.children ?? []) visit(child);
+  };
+  visit(parsed.ast);
+
+  if (parsed.frontmatter?.markset !== 0) {
+    diagnostics.push(
+      publicationDiagnostic({
+        code: "PUB003",
+        artifact: id,
+        path: artifact.path,
+        message: `The generated frontmatter for ${id} does not read as Markset version 0.`,
+        remediation:
+          "Report this as a publisher defect; the generated frontmatter must be readable by Markset's own reader.",
+      }),
+    );
+  }
+  for (const d of marksetCarrier(`${id}.md`, source).syntax) {
+    if (d.severity !== "error" || (d.location !== undefined && directiveLines.has(d.location.line))) continue;
+    diagnostics.push(
+      publicationDiagnostic({
+        code: "PUB003",
+        artifact: id,
+        path: artifact.path,
+        location: locate(d.location?.line),
+        message: `The generated document for ${id} is not valid Markset: ${d.code}, ${d.message.replace(/\.$/, "")}.`,
+        remediation:
+          "Fix the Markset syntax in the knowledge body; the document is not emitted until it parses without errors.",
+      }),
+    );
+  }
+  return diagnostics;
+}

package/src/index.ts

@@ -0,0 +1,70 @@
+/**
+ * @intentset/publisher: the audience and release projection that turns
+ * reviewed knowledge into Markset documents with provenance (Core §9,
+ * spec/publication.md).
+ */
+export { bodyLineOffset } from "./body.ts";
+export { type Chunk, type ChunkOptions, type ChunkSource, toChunks } from "./chunks.ts";
+export {
+  type GenerateOptions,
+  type GeneratedDocument,
+  PUBLICATION_KEY,
+  PUBLICATION_PROFILE,
+  type PublicationMeta,
+  type SourceStamp,
+  checkGenerated,
+  generateDocument,
+  markerLine,
+  marksetSource,
+  publicationMeta,
+} from "./generate.ts";
+export {
+  EXCLUSION_REASONS,
+  type Exclusion,
+  type ExclusionReason,
+  type ProjectOptions,
+  type Projection,
+  type Snapshot,
+  exclusionReason,
+  project,
+} from "./project.ts";
+export {
+  type PublicationIndex,
+  type PublishOptions,
+  type PublishResult,
+  type PublishedDocument,
+  REFUSAL_REASONS,
+  type RefusalReason,
+  publish,
+  renderDocument,
+} from "./publish.ts";
+export {
+  type Reference,
+  type ReferenceKind,
+  checkReferences,
+  checkTitles,
+  findReferences,
+  resolveTarget,
+  titlePattern,
+} from "./references.ts";
+export {
+  type IndexedRequest,
+  PROJECTION_ADMITS,
+  type PublicationRequest,
+  REQUEST_DIMENSIONS,
+  type RequestDimension,
+  admits,
+  checkRequest,
+  indexedRequest,
+} from "./request.ts";
+export {
+  CURRENT_PIN,
+  REVIEW_EXTENSION,
+  type ReviewRecord,
+  type ReviewStatus,
+  bindReviewPins,
+  readReviewRecord,
+  reviewSources,
+  reviewStatus,
+} from "./review.ts";
+export { YamlEmitError, type YamlValue, emitYaml } from "./yaml.ts";