@warpgogol/werkstatt-shared 0.2.0

package/src/surface/module-context.ts

@@ -0,0 +1,120 @@
+/*
+<MODULE_CONTRACT>
+<purpose>
+  RFC-0271: typed, schema-validated operating contexts for Programmatic Surface modules.
+  A module context declares which Blueprints it owns, which entitlement gates them, and which
+  locale is the canonical generation/review source before any derived public translation.
+</purpose>
+<non-goals>
+  <item>Do not read system.md or entitlements; kernel commands own I/O.</item>
+  <item>Do not redefine Blueprint axes, levels, or route policy.</item>
+</non-goals>
+</MODULE_CONTRACT>
+<CHANGE_SUMMARY>
+  <item>RFC-0271: add module context contracts for masterLocale, publishedLocales, entitlement, glossary refs, translator-note refs, and generation/approval policy.</item>
+  <item>RFC-0277: add stage and urlPolicy fields for PSEO lifecycle governance.</item>
+</CHANGE_SUMMARY>
+*/
+
+import { z } from "zod";
+
+export const moduleSiteModeSchema = z.enum(["bodenstation", "sternsystem"]);
+
+export const pseoStageSchema = z.enum(["internalCapability", "managedVisibility", "productModule"]);
+
+export const urlPolicySchema = z.enum(["nonDestruction"]);
+
+export const moduleReviewPolicySchema = z.object({
+  firstNPerTemplateField: z.number().int().min(0).optional(),
+  sampleAfterStabilization: z.number().min(0).max(1).optional(),
+  claims: z.enum(["human", "sample", "machine"]).optional(),
+});
+
+export const surfaceModuleContextSchema = z.object({
+  id: z.string().min(1).optional(),
+  entitlement: z.string().min(1),
+  blueprints: z.array(z.string().min(1)).min(1),
+  masterLocale: z.string().min(2),
+  publishedLocales: z.array(z.string().min(2)).default([]),
+  stage: pseoStageSchema.optional(),
+  urlPolicy: urlPolicySchema.optional(),
+  context: z
+    .object({
+      siteMode: moduleSiteModeSchema.optional(),
+      operatorLanguage: z.string().min(2).optional(),
+      audience: z.string().min(1).optional(),
+      forbiddenClaims: z.array(z.string().min(1)).optional(),
+      sourceBoundaries: z.array(z.string().min(1)).optional(),
+      voice: z.record(z.string(), z.unknown()).optional(),
+    })
+    .default({}),
+  indexBudget: z
+    .object({
+      publicName: z.literal("managedCoverage").optional(),
+      maxIndexable: z.number().int().min(0).optional(),
+      regionalGateDepths: z.array(z.number().int().min(0)).optional(),
+    })
+    .optional(),
+  generation: z
+    .object({
+      provider: z.string().min(1).optional(),
+      modelPolicy: z.string().min(1).optional(),
+      normalBuildCallsLlm: z.literal(false).default(false),
+    })
+    .default({ normalBuildCallsLlm: false }),
+  approval: z
+    .object({
+      requireHumanApproval: z.boolean().default(true),
+      requireReadyForTranslation: z.boolean().default(true),
+    })
+    .default({ requireHumanApproval: true, requireReadyForTranslation: true }),
+  localization: z
+    .object({
+      glossaryRefs: z.record(z.string(), z.string()).optional(),
+      translatorNoteRefs: z.record(z.string(), z.string()).optional(),
+      reviewPolicy: z.record(z.string(), moduleReviewPolicySchema).optional(),
+    })
+    .optional(),
+});
+
+export const surfaceModulesSchema = z.record(z.string(), surfaceModuleContextSchema);
+
+export type SurfaceModuleContext = z.infer<typeof surfaceModuleContextSchema> & { id: string };
+export type SurfaceModules = Record<string, SurfaceModuleContext>;
+export type PseoStage = z.infer<typeof pseoStageSchema>;
+export type UrlPolicy = z.infer<typeof urlPolicySchema>;
+
+export interface BlueprintModuleClaim {
+  blueprint: string;
+  modules: string[];
+}
+
+export function normalizeSurfaceModules(value: unknown): SurfaceModules {
+  const parsed = surfaceModulesSchema.parse(value ?? {});
+  const modules: SurfaceModules = {};
+  for (const [id, context] of Object.entries(parsed)) {
+    modules[id] = { ...context, id };
+  }
+  return modules;
+}
+
+export function findModuleForBlueprint(
+  modules: SurfaceModules,
+  blueprintId: string,
+): SurfaceModuleContext | undefined {
+  return Object.values(modules).find((module) => module.blueprints.includes(blueprintId));
+}
+
+export function findDuplicateBlueprintClaims(modules: SurfaceModules): BlueprintModuleClaim[] {
+  const claims = new Map<string, string[]>();
+  for (const module of Object.values(modules)) {
+    for (const blueprint of module.blueprints) {
+      const owners = claims.get(blueprint) ?? [];
+      owners.push(module.id);
+      claims.set(blueprint, owners);
+    }
+  }
+  return [...claims.entries()]
+    .filter(([, modulesForBlueprint]) => modulesForBlueprint.length > 1)
+    .map(([blueprint, modulesForBlueprint]) => ({ blueprint, modules: modulesForBlueprint }));
+}

package/src/surface/substance.ts

@@ -0,0 +1,161 @@
+/*
+<MODULE_CONTRACT>
+<purpose>
+  [RFC-0194] Page Substance Score: a deterministic build-time measure of whether a generated page
+  is substantively unique, not a near-duplicate template with a swapped noun. Composed from four
+  components — independent data blocks, unique-vs-family token ratio, signal blocks, and internal
+  link diversity. A page below the Blueprint's substanceMin is forced to noindex regardless of how
+  many records matched it. Pure — no I/O, no LLM.
+</purpose>
+<non-goals>
+  <item>Do not call an LLM or the network — the gate must be deterministic.</item>
+  <item>Do not mutate page content to raise a score.</item>
+</non-goals>
+</MODULE_CONTRACT>
+<CHANGE_SUMMARY>
+  <item>RFC-0194: initial scorer.</item>
+  <item>RFC-0274: walk nested visible props so substance/duplicate checks see rendered semantic text.</item>
+</CHANGE_SUMMARY>
+*/
+
+import type { PageEntry } from "./types.ts";
+
+export interface SubstanceComponents {
+  independentBlocks: number;
+  uniqueTokenRatio: number; // 0..1
+  signalBlocks: number;
+  linkDiversity: number;
+}
+
+export interface SubstanceScore {
+  value: number; // 0..100
+  components: SubstanceComponents;
+}
+
+/** Lowercase word tokens of length > 3 (cheap, deterministic, language-agnostic). */
+export function tokenize(text: string): string[] {
+  return (text.toLowerCase().match(/[a-zà-ÿ0-9]+/gi) ?? []).filter((t) => t.length > 3);
+}
+
+const NON_VISIBLE_TEXT_KEYS = new Set([
+  "id",
+  "slug",
+  "pageId",
+  "href",
+  "url",
+  "src",
+  "source",
+  "image",
+  "imageName",
+  "leadImage",
+  "backgroundImage",
+  "poster",
+  "photo",
+  "video",
+  "variant",
+  "kind",
+  "type",
+  "use",
+]);
+
+function collectVisibleText(value: unknown, key = ""): string[] {
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (!trimmed) return [];
+    if (NON_VISIBLE_TEXT_KEYS.has(key)) return [];
+    if (/^(\/|https?:|mailto:|tel:)/i.test(trimmed)) return [];
+    return [trimmed];
+  }
+  if (Array.isArray(value)) return value.flatMap((item) => collectVisibleText(item, key));
+  if (value && typeof value === "object") {
+    const out: string[] = [];
+    for (const [childKey, childValue] of Object.entries(value as Record<string, unknown>)) {
+      out.push(...collectVisibleText(childValue, childKey));
+    }
+    return out;
+  }
+  return [];
+}
+
+/** Concatenate a page's rendered semantic text, including nested visible block props. */
+export function pageText(page: PageEntry): string {
+  const parts: string[] = [page.title, page.description].filter(Boolean);
+  for (const block of page.blocks) {
+    parts.push(...collectVisibleText(block.props));
+  }
+  return parts.join(" ");
+}
+
+/** token → number of pages whose token set contains it. */
+export function buildTokenDocFreq(pages: readonly PageEntry[]): Map<string, number> {
+  const freq = new Map<string, number>();
+  for (const page of pages) {
+    const seen = new Set(tokenize(pageText(page)));
+    for (const token of seen) freq.set(token, (freq.get(token) ?? 0) + 1);
+  }
+  return freq;
+}
+
+/** Count distinct internal-link targets referenced in a page's block props. */
+function countLinkTargets(page: PageEntry): number {
+  const targets = new Set<string>();
+  const visit = (value: unknown): void => {
+    if (typeof value === "string") {
+      if (/^(\/|https?:)/.test(value)) targets.add(value);
+      return;
+    }
+    if (Array.isArray(value)) {
+      for (const item of value) visit(item);
+    } else if (value && typeof value === "object") {
+      for (const v of Object.values(value as Record<string, unknown>)) visit(v);
+    }
+  };
+  for (const block of page.blocks) visit(block.props);
+  return targets.size;
+}
+
+/**
+ * Score a page's substance. `docFreq`/`totalPages` come from the family baseline (buildTokenDocFreq);
+ * a token is "unique" when it appears in at most ceil(totalPages * 0.5) pages. `signalBlockIds` are
+ * block `type`s the Blueprint marks as substance-bearing; when omitted, any block with a non-empty
+ * `lead`/`body` counts as a signal.
+ */
+export function scoreSubstance(
+  page: PageEntry,
+  ctx: { docFreq: Map<string, number>; totalPages: number; signalBlockIds?: string[] },
+): SubstanceScore {
+  const independentBlocks = page.blocks.length;
+
+  const tokens = tokenize(pageText(page));
+  const distinct = new Set(tokens);
+  const uniqueThreshold = Math.max(1, Math.ceil(ctx.totalPages * 0.5));
+  let uniqueCount = 0;
+  for (const token of distinct) {
+    if ((ctx.docFreq.get(token) ?? 0) <= uniqueThreshold) uniqueCount += 1;
+  }
+  const uniqueTokenRatio = distinct.size === 0 ? 0 : uniqueCount / distinct.size;
+
+  const signalSet = ctx.signalBlockIds ? new Set(ctx.signalBlockIds) : null;
+  let signalBlocks = 0;
+  for (const block of page.blocks) {
+    const props = (block.props ?? {}) as Record<string, unknown>;
+    const isSignal = signalSet
+      ? signalSet.has(block.type ?? block.use ?? "")
+      : collectVisibleText(props).join(" ").length > 40;
+    if (isSignal) signalBlocks += 1;
+  }
+
+  const linkDiversity = countLinkTargets(page);
+
+  const value = Math.round(
+    Math.min(100, independentBlocks * 12) * 0.3 +
+      uniqueTokenRatio * 100 * 0.4 +
+      Math.min(100, signalBlocks * 18) * 0.2 +
+      Math.min(100, linkDiversity * 20) * 0.1,
+  );
+
+  return {
+    value,
+    components: { independentBlocks, uniqueTokenRatio, signalBlocks, linkDiversity },
+  };
+}

package/src/surface/tests/eligibility.pbt.test.ts

@@ -0,0 +1,137 @@
+import { test, expect } from "vitest";
+import fc from "fast-check";
+import {
+  normalizeSegment,
+  matchesRecord,
+  countMatching,
+  buildRecordIndex,
+  countMatchingIndexed,
+  pathKey,
+  enumerateCandidateTuples,
+  buildEligibilityMatrix,
+  type AxisFieldMap,
+} from "../eligibility.ts";
+import type { SurfaceAxis, SurfaceRecord, EligibilityPolicy } from "../types.ts";
+
+const SEGMENT_PATTERN = /^[a-z0-9-]+$/;
+
+const defaultPolicy: EligibilityPolicy = {
+  minRecordsPerDepth: { 0: 1, 1: 1, 2: 1 },
+  noindexBelowPerDepth: {},
+  redirectPolicy: "nearest-ancestor",
+  trailingSlash: true,
+  segmentPattern: SEGMENT_PATTERN,
+};
+
+test("PBT: normalizeSegment is idempotent", () => {
+  fc.assert(
+    fc.property(
+      fc.stringMatching(SEGMENT_PATTERN).map((s) => s.replace(/_/g, "-")),
+      (s) => {
+        const once = normalizeSegment(s, SEGMENT_PATTERN);
+        const twice = normalizeSegment(once, SEGMENT_PATTERN);
+        expect(twice).toBe(once);
+      },
+    ),
+  );
+});
+
+test("PBT: countMatchingIndexed matches countMatching for any tuple", () => {
+  const _axes: SurfaceAxis[] = [
+    { id: "trade", universe: ["elektriker", "friseur", "baker"] },
+    { id: "region", universe: ["berlin", "hamburg", "munich"] },
+  ];
+  const axisFieldMap: AxisFieldMap = { trade: "trade", region: "region" };
+  const records: SurfaceRecord[] = [
+    { trade: "elektriker", region: "berlin", status: "active" },
+    { trade: "elektriker", region: "hamburg", status: "active" },
+    { trade: "friseur", region: "berlin", status: "active" },
+    { trade: "friseur", region: "hamburg", status: "archived" },
+    { trade: "baker", region: "munich", status: "active" },
+  ];
+  const index = buildRecordIndex(records, axisFieldMap);
+
+  const tupleArb = fc.record({
+    trade: fc.constantFrom("elektriker", "friseur", "baker", undefined),
+    region: fc.constantFrom("berlin", "hamburg", "munich", undefined),
+  });
+
+  fc.assert(
+    fc.property(tupleArb, (tuple) => {
+      const linear = countMatching(records, tuple, axisFieldMap);
+      const indexed = countMatchingIndexed(tuple, axisFieldMap, index);
+      expect(indexed).toBe(linear);
+    }),
+  );
+});
+
+test("PBT: pathKey is deterministic for the same input", () => {
+  const axisOrder = ["a", "b", "c"];
+  const tupleArb = fc.record({
+    a: fc.constantFrom("x", "y", undefined),
+    b: fc.constantFrom("z", "w", undefined),
+    c: fc.constantFrom("q", "r", undefined),
+  });
+  const depthArb = fc.integer({ min: 0, max: 3 });
+
+  fc.assert(
+    fc.property(tupleArb, depthArb, (tuple, depth) => {
+      const k1 = pathKey(depth, tuple, axisOrder);
+      const k2 = pathKey(depth, tuple, axisOrder);
+      expect(k1).toBe(k2);
+    }),
+  );
+});
+
+test("PBT: enumerateCandidateTuples count equals product of universe sizes", () => {
+  const universeArb = fc.array(fc.stringMatching(SEGMENT_PATTERN), { minLength: 1, maxLength: 5 });
+  const axesArb = fc.array(
+    fc.record({ id: fc.stringMatching(SEGMENT_PATTERN), universe: universeArb }),
+    { minLength: 1, maxLength: 3 },
+  );
+
+  fc.assert(
+    fc.property(axesArb, (axes) => {
+      const depth = axes.length;
+      const tuples = [...enumerateCandidateTuples(axes, depth)];
+      const expected = axes.reduce((acc, a) => acc * a.universe.length, 1);
+      expect(tuples).toHaveLength(expected);
+    }),
+  );
+});
+
+test("PBT: buildEligibilityMatrix entries are ordered by depth", () => {
+  const axes: SurfaceAxis[] = [
+    { id: "trade", universe: ["elektriker", "friseur"] },
+    { id: "region", universe: ["berlin", "hamburg"] },
+  ];
+  const axisFieldMap: AxisFieldMap = { trade: "trade", region: "region" };
+  const records: SurfaceRecord[] = [
+    { trade: "elektriker", region: "berlin", status: "active" },
+    { trade: "friseur", region: "hamburg", status: "active" },
+  ];
+
+  fc.assert(
+    fc.property(fc.boolean(), (forceGate) => {
+      const matrix = buildEligibilityMatrix(axes, axisFieldMap, records, defaultPolicy, {
+        forceNonIndexableDepths: forceGate ? [2] : [],
+      });
+      for (let i = 1; i < matrix.entries.length; i++) {
+        expect(matrix.entries[i]!.depth).toBeGreaterThanOrEqual(matrix.entries[i - 1]!.depth);
+      }
+    }),
+  );
+});
+
+test("PBT: matchesRecord with empty tuple always returns true", () => {
+  const recordArb: fc.Arbitrary<SurfaceRecord> = fc.record({
+    field: fc.string(),
+    status: fc.constantFrom("active", "archived"),
+  });
+
+  fc.assert(
+    fc.property(recordArb, (record) => {
+      expect(matchesRecord(record, {}, {})).toBe(true);
+    }),
+  );
+});