@fungi.computer/forage 0.1.0 → 0.2.1

package/dist/rendering.js

@@ -1,67 +1,71 @@
 import { z } from "zod";
 import { ForageError } from "./errors.js";
-import { canonicalStringify, compareCodeUnits, isJsonObject, jsonValueSchema, } from "./normalization.js";
-import { maximumSampleBytes, maximumSampleDepth, maximumSampleEntries, } from "./schemas.js";
-const schemaNode = z.lazy(() => z.union([
-    z.boolean(),
+import { compareCodeUnits, jsonValueSchema } from "./normalization.js";
+/** The JSON Schema subset Forage reads when it renders and samples a schema. */
+const schemaNode = z.union([
     z.looseObject({
         $ref: z.string().optional(),
-        $defs: z.record(z.string(), schemaNode).optional(),
-        definitions: z.record(z.string(), schemaNode).optional(),
+        $defs: z.record(z.string(), z.unknown()).optional(),
+        definitions: z.record(z.string(), z.unknown()).optional(),
         const: z.unknown().optional(),
         enum: z.array(z.unknown()).optional(),
-        oneOf: z.array(schemaNode).optional(),
-        anyOf: z.array(schemaNode).optional(),
-        allOf: z.array(schemaNode).optional(),
+        oneOf: z.array(z.unknown()).optional(),
+        anyOf: z.array(z.unknown()).optional(),
+        allOf: z.array(z.unknown()).optional(),
         type: z.union([z.string(), z.array(z.string())]).optional(),
-        properties: z.record(z.string(), schemaNode).optional(),
-        additionalProperties: schemaNode.optional(),
+        properties: z.record(z.string(), z.unknown()).optional(),
+        additionalProperties: z.unknown().optional(),
         required: z.array(z.string()).optional(),
-        prefixItems: z.array(schemaNode).optional(),
-        items: z.union([schemaNode, z.array(schemaNode)]).optional(),
+        prefixItems: z.array(z.unknown()).optional(),
+        items: z.unknown().optional(),
         examples: z.array(z.unknown()).optional(),
         default: z.unknown().optional(),
     }),
-]));
-/**
- * Facts already derived, per schema object and direction. A Standard JSON
- * Schema describes an immutable schema, so its projection is a pure function
- * of the object: compiling a catalog asks for the same projection several
- * times per operation (size, fingerprint, sample, rendering), and a host that
- * recompiles a catalog over the same installed schemas asks again on every
- * compile. Projecting, validating, fingerprinting and sizing once per schema
- * keeps catalog compilation proportional to what actually changed.
- */
-const schemaFacts = new WeakMap();
-function factsFor(schema, io) {
-    let entry = schemaFacts.get(schema);
+    z.boolean(),
+]);
+/** Standard JSON Schema projections are immutable, per source and direction. */
+const projections = new WeakMap();
+function projectionFor(schema, io) {
+    let entry = projections.get(schema);
     const cached = entry?.[io];
     if (cached !== undefined)
         return cached;
-    let node;
+    let raw;
     try {
-        node = schemaNode.parse(schema["~standard"].jsonSchema[io]({ target: "draft-2020-12" }));
+        raw = schema["~standard"].jsonSchema[io]({ target: "draft-2020-12" });
     }
     catch {
         throw new ForageError("invalid_input", "Schema cannot be projected to JSON Schema");
     }
-    const json = jsonValueSchema.safeParse(node);
-    const facts = {
-        node,
-        json: json.success ? json.data : undefined,
-    };
+    decodeNode(raw);
+    const projection = { raw };
     if (entry === undefined) {
         entry = {};
-        schemaFacts.set(schema, entry);
+        projections.set(schema, entry);
     }
-    entry[io] = facts;
-    return facts;
+    entry[io] = projection;
+    return projection;
 }
-function schemaFor(schema, io) {
-    return factsFor(schema, io).node;
+/** The producer's document, without a recursive projection decode or copy. */
+export function rawSchema(schema, io) {
+    return projectionFor(schema, io).raw;
 }
-function isNodeList(value) {
-    return Array.isArray(value);
+/** Native serialization owns the document's text representation and failures. */
+export function schemaText(schema, io) {
+    const projection = projectionFor(schema, io);
+    try {
+        projection.text ??= JSON.stringify(projection.raw);
+        return projection.text;
+    }
+    catch {
+        throw new ForageError("invalid_input", "Schema cannot be projected to JSON Schema");
+    }
+}
+function decodeNode(value) {
+    const parsed = schemaNode.safeParse(value);
+    if (!parsed.success)
+        throw new ForageError("invalid_input", "Schema cannot be projected to JSON Schema");
+    return parsed.data;
 }
 /** Bounds the work and size of expanding shared `$defs` in one description. */
 const maximumReferenceExpansions = 256;
@@ -77,7 +81,7 @@ function renderReference(reference, depth, refs) {
     refs.budget.expansions += 1;
     if (refs.budget.expansions > maximumReferenceExpansions)
         throw new ForageError("invalid_input", `Schema references expand more than ${String(maximumReferenceExpansions)} times; flatten its shared definitions`);
-    return renderNode(target, depth, {
+    return renderNode(decodeNode(target), depth, {
         definitions: refs.definitions,
         expanding: new Set([...refs.expanding, name]),
         budget: refs.budget,
@@ -94,7 +98,7 @@ function rendersUnion(node, refs) {
             : refs.definitions[name];
         return (name !== undefined &&
             target !== undefined &&
-            rendersUnion(target, {
+            rendersUnion(decodeNode(target), {
                 definitions: refs.definitions,
                 expanding: new Set([...refs.expanding, name]),
                 budget: refs.budget,
@@ -106,10 +110,17 @@ function rendersUnion(node, refs) {
         (node.oneOf !== undefined && node.oneOf.length > 1) ||
         (node.anyOf !== undefined && node.anyOf.length > 1));
 }
-function renderNode(node, depth, refs) {
-    if (node === undefined || node === true || node === false || depth > 12) {
+function renderNode(node, depth, scope) {
+    if (depth > 12) {
         return "unknown";
     }
+    if (node === true || node === false)
+        return "unknown";
+    const refs = scope ?? {
+        definitions: { ...node.definitions, ...node.$defs },
+        expanding: new Set(),
+        budget: { expansions: 0 },
+    };
     if (node.$ref)
         return renderReference(node.$ref, depth, refs);
     if (node.const !== undefined)
@@ -118,20 +129,21 @@ function renderNode(node, depth, refs) {
         return node.enum.map((value) => JSON.stringify(value)).join(" | ");
     if (node.oneOf) {
         return node.oneOf
-            .map((value) => renderNode(value, depth + 1, refs))
+            .map((value) => renderNode(decodeNode(value), depth + 1, refs))
             .join(" | ");
     }
     if (node.anyOf) {
         return node.anyOf
-            .map((value) => renderNode(value, depth + 1, refs))
+            .map((value) => renderNode(decodeNode(value), depth + 1, refs))
             .join(" | ");
     }
     if (node.allOf) {
         return node.allOf
             .map((value) => {
-            const member = renderNode(value, depth + 1, refs);
+            const child = decodeNode(value);
+            const member = renderNode(child, depth + 1, refs);
             // `&` binds tighter than `|`: keep a union member whole.
-            return rendersUnion(value, refs) ? `(${member})` : member;
+            return rendersUnion(child, refs) ? `(${member})` : member;
         })
             .join(" & ");
     }
@@ -142,13 +154,13 @@ function renderNode(node, depth, refs) {
             if (additional === false)
                 return "Record<string, never>";
             if (additional && additional !== true) {
-                return `Record<string, ${renderNode(additional, depth + 1, refs)}>`;
+                return `Record<string, ${renderNode(decodeNode(additional), depth + 1, refs)}>`;
             }
             return "Record<string, unknown>";
         }
         const required = new Set(node.required ?? []);
         const lines = properties.map(([key, value]) => {
-            const child = renderNode(value, depth + 1, refs);
+            const child = renderNode(decodeNode(value), depth + 1, refs);
             const optional = required.has(key) ? "" : "?";
             const safeKey = /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)
                 ? key
@@ -162,21 +174,21 @@ ${"  ".repeat(depth)}}`;
     if (node.type === "array") {
         const items = node.items;
         if (node.prefixItems && node.prefixItems.length > 0) {
-            const tuple = node.prefixItems.map((value) => renderNode(value, depth + 1, refs));
-            if (items !== undefined && items !== true && !isNodeList(items)) {
-                tuple.push(`...${renderNode(items, depth + 1, refs)}[]`);
+            const tuple = node.prefixItems.map((value) => renderNode(decodeNode(value), depth + 1, refs));
+            if (items !== undefined && items !== true && !Array.isArray(items)) {
+                tuple.push(`...${renderNode(decodeNode(items), depth + 1, refs)}[]`);
             }
             return `[${tuple.join(", ")}]`;
         }
-        if (items !== undefined && isNodeList(items))
-            return `[${items.map((value) => renderNode(value, depth + 1, refs)).join(", ")}][]`;
-        const item = renderNode(items, depth + 1, refs);
-        const compoundItem = items !== undefined &&
-            items !== true &&
-            items !== false &&
-            (items.oneOf !== undefined ||
-                items.anyOf !== undefined ||
-                items.allOf !== undefined);
+        if (items !== undefined && Array.isArray(items))
+            return `[${items.map((value) => renderNode(decodeNode(value), depth + 1, refs)).join(", ")}][]`;
+        const itemNode = items === undefined ? true : decodeNode(items);
+        const item = renderNode(itemNode, depth + 1, refs);
+        const compoundItem = itemNode !== true &&
+            itemNode !== false &&
+            (itemNode.oneOf !== undefined ||
+                itemNode.anyOf !== undefined ||
+                itemNode.allOf !== undefined);
         return `${compoundItem ? `(${item})` : item}[]`;
     }
     switch (node.type) {
@@ -195,15 +207,7 @@ ${"  ".repeat(depth)}}`;
 }
 /** Render one schema's JSON Schema as deterministic TypeScript type text. */
 function renderSchemaType(schema, io = "input") {
-    const root = schemaFor(schema, io);
-    const definitions = root === true || root === false
-        ? {}
-        : { ...root.definitions, ...root.$defs };
-    return renderNode(root, 0, {
-        definitions,
-        expanding: new Set(),
-        budget: { expansions: 0 },
-    });
+    return renderNode(decodeNode(rawSchema(schema, io)), 0);
 }
 function safeTypeName(name) {
     return name.replace(/[^A-Za-z0-9_$]/g, "_");
@@ -228,33 +232,38 @@ export function renderOperationTypes(operations) {
         ? types
         : `${types}\nexport declare const env: ${operations.map((operation) => operation.environment).join(" & ")};`;
 }
-/** Prepare all TypeScript descriptions for one operation at installation. */
+const renderedOperations = new WeakMap();
+/** Render only requested operations, once per immutable prepared operation. */
 export function renderOperation(operation) {
+    const cached = renderedOperations.get(operation);
+    if (cached !== undefined)
+        return cached;
     const name = safeTypeName(operation.path.replace(/\./g, "$"));
     const input = renderSchemaType(operation.input, "input");
     const output = operation.output
         ? renderSchemaType(operation.output, "output")
         : "unknown";
-    return Object.freeze({
+    const rendered = Object.freeze({
         indexEnvironment: renderEnvironment(operation.path, input, output),
         environment: renderEnvironment(operation.path, `${name}_Input`, `${name}_Output`),
         types: `export type ${name}_Input = ${input};\nexport type ${name}_Output = ${output};`,
     });
+    renderedOperations.set(operation, rendered);
+    return rendered;
 }
 /** Derive sample arguments solely from input-schema examples and defaults. */
 export function deriveSample(schema) {
-    const node = schemaFor(schema, "input");
+    const node = decodeNode(rawSchema(schema, "input"));
     if (node === true || node === false || !node.properties) {
         return undefined;
     }
-    const properties = node.properties;
     const sample = {};
     let found = false;
-    const entries = Object.entries(properties).sort(([left], [right]) => compareCodeUnits(left, right));
+    const entries = Object.entries(node.properties).sort(([left], [right]) => compareCodeUnits(left, right));
     for (const [key, value] of entries) {
-        if (value === true || value === false)
+        const property = decodeNode(value);
+        if (property === true || property === false)
             continue;
-        const property = value;
         const examples = property.examples;
         if (examples && examples.length > 0) {
             const example = jsonValueSchema.safeParse(examples[0]);
@@ -273,69 +282,3 @@ export function deriveSample(schema) {
     }
     return found ? sample : undefined;
 }
-/** Preserve validator constraints in the operation's Standard JSON Schema. */
-export function inputSchemaProjection(schema) {
-    const projection = factsFor(schema, "input").json;
-    if (projection === undefined) {
-        throw new ForageError("invalid_input", "Schema cannot be projected to JSON Schema");
-    }
-    return projection;
-}
-/** Produce a canonical schema representation used in peer identity and stamps. */
-export function schemaFingerprint(schema, io = "input") {
-    const facts = factsFor(schema, io);
-    facts.fingerprint ??= canonicalStringify(facts.json ?? null);
-    return facts.fingerprint;
-}
-function schemaDepth(value, depth) {
-    if (depth > maximumSampleDepth)
-        return depth;
-    if (Array.isArray(value)) {
-        let maximum = depth;
-        for (const item of value) {
-            maximum = Math.max(maximum, schemaDepth(item, depth + 1));
-        }
-        return maximum;
-    }
-    if (!isJsonObject(value))
-        return depth;
-    let maximum = depth;
-    for (const entry of Object.values(value)) {
-        maximum = Math.max(maximum, schemaDepth(entry, depth + 1));
-    }
-    return maximum;
-}
-/** Validate the bounded JSON projection before a catalog is installed. */
-export function schemaProjectionSize(schema, io) {
-    const facts = factsFor(schema, io);
-    facts.size ??= { bytes: measureProjection(schema, io, facts.json) };
-    return facts.size.bytes;
-}
-function measureProjection(schema, io, projection) {
-    if (!projection)
-        return undefined;
-    const encoded = JSON.stringify(projection);
-    if (encoded.length > 20_000 ||
-        schemaDepth(projection, 0) > maximumSampleDepth) {
-        return undefined;
-    }
-    if (io === "output")
-        return encoded.length;
-    const sample = deriveSample(schema);
-    if (!sample)
-        return encoded.length;
-    if (Object.keys(sample).length > maximumSampleEntries ||
-        schemaDepth(sample, 0) > maximumSampleDepth) {
-        return undefined;
-    }
-    let sampleText;
-    try {
-        sampleText = JSON.stringify(sample);
-    }
-    catch {
-        throw new ForageError("invalid_input", "Derived schema sample cannot be serialized");
-    }
-    if (!sampleText || sampleText.length > maximumSampleBytes)
-        return undefined;
-    return encoded.length + sampleText.length;
-}

package/dist/schemas.d.ts

@@ -3,6 +3,7 @@ export declare const maximumDescriptionTypes = 20000;
 export declare const maximumSampleEntries = 100;
 export declare const maximumSampleBytes = 4000;
 export declare const maximumSampleDepth = 16;
+export declare function jsonDepth(value: z.output<z.ZodJSONSchema>, depth: number): number;
 /** Input accepted by the pure catalog search seam. */
 export declare const forageSearchInput: z.ZodLazy<z.ZodObject<{
     query: z.ZodString;

package/dist/schemas.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { isJsonObject } from "./normalization.js";
 const pathPattern = /^[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*)+$/;
 const maximumCatalogOperations = 1_000;
 const maximumScore = 221;
@@ -6,7 +7,7 @@ export const maximumDescriptionTypes = 20_000;
 export const maximumSampleEntries = 100;
 export const maximumSampleBytes = 4_000;
 export const maximumSampleDepth = 16;
-function jsonDepth(value, depth) {
+export function jsonDepth(value, depth) {
     if (depth > maximumSampleDepth)
         return depth;
     if (Array.isArray(value)) {
@@ -16,13 +17,10 @@ function jsonDepth(value, depth) {
         }
         return maximum;
     }
-    if (value === null)
-        return depth;
-    const record = z.record(z.string(), z.json()).safeParse(value);
-    if (!record.success)
+    if (!isJsonObject(value))
         return depth;
     let maximum = depth;
-    for (const item of Object.values(record.data)) {
+    for (const item of Object.values(value)) {
         maximum = Math.max(maximum, jsonDepth(item, depth + 1));
     }
     return maximum;

package/dist/search.d.ts

@@ -1,12 +1,11 @@
 import { type Operation, type SearchOutput } from "./schemas.js";
-import { type RenderedOperation } from "./rendering.js";
 import type { JsonValue } from "./normalization.js";
 /** Private installation-time projection used by the pure ranking function. */
 export type PreparedOperation = Operation & {
     readonly fingerprint: string;
-    readonly inputSchema: JsonValue;
     readonly sample: Record<string, JsonValue> | undefined;
-    readonly rendered: RenderedOperation;
+    readonly inputBytes: number;
+    readonly outputBytes: number;
 };
 /** Prepare schema-derived search metadata once for an immutable catalog revision. */
 export declare function prepareOperation(operation: Operation): PreparedOperation;

package/dist/search.js

@@ -1,15 +1,17 @@
 import { searchOutputSchema, } from "./schemas.js";
-import { canonicalStringify, compareCodeUnits, hashString64, isStopword, normalizeQuery, operationMethod, operationNamespace, stemVariants, tokenize, } from "./normalization.js";
-import { deriveSample, inputSchemaProjection, renderOperation, schemaFingerprint, } from "./rendering.js";
+import { compareCodeUnits, hashString64, isStopword, normalizeQuery, operationMethod, operationNamespace, stemVariants, tokenize, } from "./normalization.js";
+import { deriveSample, schemaText } from "./rendering.js";
 import { ForageError } from "./errors.js";
 /** Prepare schema-derived search metadata once for an immutable catalog revision. */
 export function prepareOperation(operation) {
+    const input = schemaText(operation.input, "input");
+    const output = operation.output ? schemaText(operation.output, "output") : "";
     return Object.freeze({
         ...operation,
-        fingerprint: schemaFingerprint(operation.input),
-        inputSchema: inputSchemaProjection(operation.input),
+        fingerprint: hashString64(input + "\n" + output),
+        inputBytes: input.length,
+        outputBytes: output.length,
         sample: deriveSample(operation.input),
-        rendered: renderOperation(operation),
     });
 }
 const matchWeight = {
@@ -21,33 +23,33 @@ const matchWeight = {
     substring: 2,
     includes: 1,
 };
-function fieldWeight(field) {
-    switch (field) {
-        case "path":
-            return 12;
-        case "namespace":
-            return 8;
-        case "method":
-            return 10;
-        case "description":
-            return 5;
-        default:
-            return 5;
-    }
-}
+const fieldWeight = {
+    path: 12,
+    namespace: 8,
+    method: 10,
+    description: 5,
+};
+const fullCoverageBonus = 25;
+const partialCoverageBonus = 10;
+const firstTokenBonus = 8;
+const exactMatchBonus = 20;
+const maxBonus = Math.max(fullCoverageBonus, partialCoverageBonus) +
+    firstTokenBonus +
+    exactMatchBonus;
+const dominanceDivisor = 1 +
+    Math.max(...Object.values(matchWeight)) *
+        Math.max(...Object.values(fieldWeight)) +
+    maxBonus;
 /** Identify every model-visible part of the catalog, including schemas. */
 function catalogStamp(operations) {
     const lines = [...operations]
         .sort((left, right) => compareCodeUnits(left.path, right.path))
-        .map((operation) => canonicalStringify([
+        .map((operation) => JSON.stringify([
         operation.path,
         operation.description ?? null,
         operation.capabilityNotes ?? null,
         operation.instructions ?? null,
         operation.fingerprint,
-        operation.output === undefined
-            ? null
-            : schemaFingerprint(operation.output, "output"),
     ]));
     return hashString64(lines.join("\n"));
 }
@@ -74,7 +76,7 @@ function scoreField(rawValue, field, queryTokens) {
     const value = normalizeQuery(rawValue);
     let bestLevel = "none";
     let bestScore = 0;
-    let matchedTokens = [];
+    const matchedTokens = [];
     for (const token of queryTokens) {
         const terms = [token, ...stemVariants(token)];
         let selected = "none";
@@ -87,14 +89,12 @@ function scoreField(rawValue, field, queryTokens) {
         }
         if (selected === "none")
             continue;
-        const score = matchWeight[selected] * fieldWeight(field);
+        const score = matchWeight[selected] * fieldWeight[field];
+        if (!matchedTokens.includes(token))
+            matchedTokens.push(token);
         if (score > bestScore) {
             bestLevel = selected;
             bestScore = score;
-            matchedTokens = [token];
-        }
-        else if (score === bestScore && !matchedTokens.includes(token)) {
-            matchedTokens.push(token);
         }
     }
     return { level: bestLevel, score: bestScore, matchedTokens };
@@ -139,7 +139,7 @@ function emptyResult(stamp) {
         catalogStamp: stamp,
     });
 }
-function scoreCandidate(operation, normalizedQuery, queryTokens, coverageTokens, threshold) {
+function scoreCandidate(operation, queryTokens, coverageTokens) {
     const namespace = operationNamespace(operation.path);
     const method = operationMethod(operation.path);
     const description = operation.description ?? "";
@@ -149,23 +149,27 @@ function scoreCandidate(operation, normalizedQuery, queryTokens, coverageTokens,
     const methodScore = scoreField(method, "method", queryTokens);
     const descriptionScore = scoreField(description, "description", queryTokens);
     const matched = coverageTokens.filter((token) => fields.some((field) => tokenMatches(field, token))).length;
-    const coverage = matched / coverageTokens.length;
-    const phraseMatch = normalizeQuery(`${operation.path} ${description}`).includes(normalizedQuery);
-    if (coverage < threshold && !phraseMatch)
+    if (matched === 0)
         return undefined;
+    const coverage = matched / coverageTokens.length;
     const baseScore = Math.max(pathScore.score, namespaceScore.score, methodScore.score, descriptionScore.score);
     if (baseScore === 0)
         return undefined;
-    let bonus = matched === coverageTokens.length ? 25 : coverage * 10;
+    let bonus = matched === coverageTokens.length
+        ? fullCoverageBonus
+        : coverage * partialCoverageBonus;
     const firstToken = coverageTokens[0];
     if (firstToken && fields.some((field) => tokenMatches(field, firstToken))) {
-        bonus += 8;
+        bonus += firstTokenBonus;
+    }
+    if (pathScore.level === "exact" || methodScore.level === "exact") {
+        bonus += exactMatchBonus;
     }
-    if (pathScore.level === "exact" || methodScore.level === "exact")
-        bonus += 20;
     const result = {
         path: operation.path,
-        score: baseScore + bonus,
+        // A whole matched word outranks any field-quality difference; the
+        // derived divisor keeps all field and bonus evidence below one word.
+        score: matched + (baseScore + bonus) / dominanceDivisor,
     };
     if (description)
         result.description = description;
@@ -246,10 +250,9 @@ export function searchCatalog(operations, query, maxResults = 50, stamp) {
     const coverageTokens = queryTokens.filter((token) => !isStopword(token));
     if (coverageTokens.length === 0)
         return emptyResult(stamp);
-    const threshold = coverageTokens.length <= 2 ? 1 : 0.6;
     const candidates = [];
     for (const operation of operations) {
-        const candidate = scoreCandidate(operation, normalizedQuery, queryTokens, coverageTokens, threshold);
+        const candidate = scoreCandidate(operation, queryTokens, coverageTokens);
         if (candidate)
             candidates.push(candidate);
     }