@segmentstream/ontology 0.0.0-stage → 0.92.1

package/dist/service/discovery.js

@@ -0,0 +1,436 @@
+import { createHash } from "node:crypto";
+import { Data } from "effect";
+import { encodeManifest } from "../manifest.js";
+import { effectivePermissions, isAllowed, permissionOf, READ_VERB, } from "../permissions.js";
+import { metricDescription, metricsFor, modelMetricDescription, modelMetricsFor, } from "../render.js";
+import { compareRefs, normalizeSearch, searchDocuments, } from "./discovery-search.js";
+import { visibleTools } from "./mcp/tools.js";
+export const DISCOVERY_RESULT_BYTES = 16 * 1024;
+/** Expected metadata errors retain their public reason without exposing excluded definitions. */
+export class DiscoveryError extends Data.TaggedError("DiscoveryError") {
+}
+function object(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function digest(value) {
+    return createHash("sha256").update(JSON.stringify(value)).digest("hex");
+}
+function preview(text, maximum) {
+    return Array.from(text).slice(0, maximum).join("");
+}
+function summary(ref, name, label, description) {
+    const shortLabel = preview(label, 120);
+    const shortDescription = preview(description, 240);
+    return {
+        ref,
+        name,
+        label: shortLabel,
+        description: shortDescription,
+        summary_truncated: shortLabel !== label || shortDescription !== description,
+    };
+}
+/** At this probability or above, the decision model thinks the request concerns an entity. */
+const RELEVANT = 0.5;
+/** A ranking the model has not answered by then falls back to word matching. */
+const DECISION_TIMEOUT_MS = 8000;
+/** One yes-or-no question for the decision model: whether the request concerns this entity. */
+function concerns(entity) {
+    const description = preview(entity.description, 240);
+    return (`Does the request concern ${entity.label} records (${entity.name})?` +
+        (description === "" ? "" : ` ${entity.label}: ${description}`));
+}
+/** Exactly the successful MCP result envelope, including JSON's escaped text representation. */
+export function discoveryResultBytes(result) {
+    return Buffer.byteLength(JSON.stringify({
+        content: [{ type: "text", text: JSON.stringify(result) }],
+        structuredContent: result,
+    }));
+}
+function checkHash(hash, expected) {
+    if (expected !== undefined && expected !== hash) {
+        throw new DiscoveryError({
+            error: "ontology_version_mismatch",
+            detail: "The ontology changed. Repeat the relevant discovery or inspection with the current hash.",
+            hash,
+        });
+    }
+}
+function invalidCursor() {
+    throw new DiscoveryError({
+        error: "invalid_cursor",
+        detail: "Restart this index or search; its cursor is invalid.",
+    });
+}
+function page(input) {
+    const scope = digest([...new Set(input.permissions)].sort(compareRefs));
+    const request = digest(input.request);
+    let start = 0;
+    if (input.after !== undefined) {
+        let cursor;
+        try {
+            if (!/^[A-Za-z0-9_-]+$/.test(input.after)) {
+                return invalidCursor();
+            }
+            cursor = JSON.parse(Buffer.from(input.after, "base64url").toString("utf8"));
+        }
+        catch (error) {
+            if (error instanceof DiscoveryError) {
+                throw error;
+            }
+            return invalidCursor();
+        }
+        if (!object(cursor) || typeof cursor.hash !== "string") {
+            return invalidCursor();
+        }
+        checkHash(input.hash, cursor.hash);
+        if (cursor.scope !== scope || cursor.request !== request || typeof cursor.last !== "string") {
+            return invalidCursor();
+        }
+        const position = input.items.findIndex(({ ref }) => ref === cursor.last);
+        if (position < 0) {
+            return invalidCursor();
+        }
+        start = position + 1;
+    }
+    const cursorAfter = (last) => Buffer.from(JSON.stringify({ hash: input.hash, scope, request, last })).toString("base64url");
+    const selected = [];
+    let result = input.result(selected, null);
+    for (const item of input.items.slice(start, start + input.limit)) {
+        const candidate = [...selected, item];
+        const more = start + candidate.length < input.items.length;
+        const next = input.result(candidate, more ? cursorAfter(item.ref) : null);
+        if (discoveryResultBytes(next) > DISCOVERY_RESULT_BYTES) {
+            if (selected.length === 0) {
+                throw new DiscoveryError({
+                    error: "discovery_too_large",
+                    detail: "This discovery entry exceeds the response budget.",
+                });
+            }
+            break;
+        }
+        selected.push(item);
+        result = next;
+    }
+    return result;
+}
+function textFields(value, prefix = "", weight = 10) {
+    return ["name", "label", "plural", "description"].flatMap((key) => {
+        const text = value[key];
+        return typeof text === "string"
+            ? [{ path: prefix + key, text, weight: key === "description" ? weight : weight + 10 }]
+            : [];
+    });
+}
+/** JSON Schema field descriptions, including nested properties and local definitions. */
+function inputFields(schema, prefix = "input") {
+    const fields = [];
+    for (const [key, value] of Object.entries(schema)) {
+        if (key === "description" && typeof value === "string") {
+            fields.push({ path: `${prefix}.description`, text: value, weight: 10 });
+        }
+        else if (key === "properties" && object(value)) {
+            for (const [name, property] of Object.entries(value)) {
+                fields.push({ path: `${prefix}.properties.${name}.name`, text: name, weight: 20 });
+                if (object(property)) {
+                    fields.push(...inputFields(property, `${prefix}.properties.${name}`));
+                }
+            }
+        }
+        else if (object(value)) {
+            fields.push(...inputFields(value, `${prefix}.${key}`));
+        }
+        else if (Array.isArray(value)) {
+            value.forEach((branch, index) => {
+                if (object(branch)) {
+                    fields.push(...inputFields(branch, `${prefix}.${key}.${index}`));
+                }
+            });
+        }
+    }
+    return fields;
+}
+/** Immutable deployment metadata; authorization and projections are recomputed for every caller. */
+export function createDiscovery(built, tools) {
+    const { manifest } = built.plan.ontology;
+    const wire = encodeManifest(manifest);
+    const wireEntities = new Map(wire.entities.map((entity) => [entity.name, entity]));
+    const actionDefinitions = new Map();
+    const incoming = new Map();
+    for (const source of manifest.entities) {
+        source.relationships.forEach((relationship, index) => {
+            const edges = incoming.get(relationship.to) ?? [];
+            edges.push({ source, relationship, index });
+            incoming.set(relationship.to, edges);
+        });
+    }
+    for (const tool of tools) {
+        if (tool.target.kind !== "action") {
+            continue;
+        }
+        const { planned, action } = tool.target;
+        actionDefinitions.set(`action:${planned.name}.${action.name}`, {
+            name: action.name,
+            entity: { ref: `entity:${planned.name}` },
+            tool_name: tool.name,
+            generated: action.generated,
+            description: action.action.description,
+            tool_description: tool.description,
+            permissions: tool.permissions,
+            input_schema: tool.inputSchema,
+            annotations: tool.annotations,
+            connections: action.action.connections.map(({ key, description }) => ({ key, description })),
+        });
+    }
+    function projection(principal, caller) {
+        const permissions = effectivePermissions(principal);
+        const holder = { permissions };
+        const visibleEntities = new Map(manifest.entities
+            .filter((entity) => entity.source === undefined ||
+            isAllowed(entity.permissions?.read ?? permissionOf(entity.name, READ_VERB), holder))
+            .map((entity) => [entity.name, entity]));
+        const actions = new Map();
+        for (const tool of visibleTools(tools, principal, caller)) {
+            if (tool.target.kind === "action") {
+                const ref = `action:${tool.target.planned.name}.${tool.target.action.name}`;
+                actions.set(ref, actionDefinitions.get(ref));
+            }
+        }
+        const metrics = new Map();
+        for (const { name, entity, metric } of metricsFor(manifest, permissions)) {
+            const encoded = wireEntities.get(entity.name)?.metrics;
+            metrics.set(`metric:${name}`, {
+                ...encoded.find((candidate) => candidate.name === metric.name),
+                name,
+                entity: { ref: `entity:${entity.name}` },
+                explanation: metricDescription(entity, metric),
+            });
+        }
+        const model = wire.metrics;
+        for (const metric of modelMetricsFor(manifest, permissions)) {
+            metrics.set(`metric:${metric.name}`, {
+                ...model?.find((candidate) => candidate.name === metric.name),
+                explanation: modelMetricDescription(metric),
+                input_refs: metric.inputs.map((name) => `metric:${name}`),
+                computation: "Computed on the combined results of its input entity metrics; " +
+                    "division by zero and missing inputs remain null.",
+            });
+        }
+        function relationships(entity) {
+            const declared = entity.relationships.map((relationship, index) => ({
+                ...(wireEntities.get(entity.name)?.relationships)[index],
+                direction: "declared",
+                target: {
+                    ref: `entity:${relationship.to}`,
+                    inspectable: visibleEntities.has(relationship.to),
+                },
+            }));
+            const inverse = (incoming.get(entity.name) ?? []).flatMap(({ source, relationship, index }) => {
+                const encoded = (wireEntities.get(source.name)?.relationships)[index];
+                return [
+                    {
+                        name: relationship.inverse,
+                        inverse: relationship.name,
+                        to: source.name,
+                        cardinality: "many",
+                        optional: true,
+                        direction: "inverse",
+                        description: relationship.description,
+                        phrase: [relationship.phrase[1], relationship.phrase[0]],
+                        declaration: { entity: { ref: `entity:${source.name}` }, relationship: encoded },
+                        ...(relationship.on === undefined
+                            ? {}
+                            : {
+                                on: {
+                                    property: relationship.on.targetProperty,
+                                    target_property: relationship.on.property,
+                                },
+                            }),
+                        target: {
+                            ref: `entity:${source.name}`,
+                            inspectable: visibleEntities.has(source.name),
+                        },
+                    },
+                ];
+            });
+            return [...declared, ...inverse];
+        }
+        function documents() {
+            const documents = [];
+            for (const entity of visibleEntities.values()) {
+                const fields = textFields(entity, "", 50);
+                entity.properties.forEach((property) => {
+                    fields.push(...textFields(property, `properties.${property.name}.`));
+                    property.options?.forEach((option, index) => {
+                        fields.push(...textFields(option, `properties.${property.name}.options.${index}.`));
+                        fields.push({
+                            path: `properties.${property.name}.options.${index}.value`,
+                            text: option.value,
+                            weight: 20,
+                        });
+                    });
+                });
+                for (const relationship of relationships(entity)) {
+                    const prefix = `relationships.${relationship.name}.`;
+                    fields.push(...textFields(relationship, prefix));
+                    relationship.phrase.forEach((text, index) => {
+                        fields.push({ path: `${prefix}phrase.${index}`, text, weight: 10 });
+                    });
+                    const target = visibleEntities.get(relationship.to);
+                    if (target !== undefined) {
+                        fields.push({ path: `${prefix}target.label`, text: target.label, weight: 10 });
+                    }
+                }
+                documents.push({
+                    summary: {
+                        ...summary(`entity:${entity.name}`, entity.name, entity.label, entity.description),
+                        kind: "entity",
+                        entity: entity.name,
+                    },
+                    fields,
+                });
+            }
+            for (const [ref, action] of actions) {
+                const owner = action.entity.ref.slice("entity:".length);
+                const name = `${owner}.${action.name}`;
+                documents.push({
+                    summary: {
+                        ...summary(ref, name, name, action.description),
+                        kind: "action",
+                        entity: owner,
+                    },
+                    fields: [
+                        ...textFields({ ...action, name }, "", 50),
+                        ...inputFields(action.input_schema),
+                    ],
+                });
+            }
+            for (const [ref, metric] of metrics) {
+                const owner = object(metric.entity)
+                    ? metric.entity.ref.slice("entity:".length)
+                    : undefined;
+                documents.push({
+                    summary: {
+                        ...summary(ref, metric.name, metric.label, metric.description),
+                        kind: "metric",
+                        ...(owner === undefined ? {} : { entity: owner }),
+                    },
+                    fields: textFields(metric, "", 50),
+                });
+            }
+            return documents;
+        }
+        return {
+            permissions,
+            visibleEntities,
+            actions,
+            metrics,
+            documents,
+        };
+    }
+    return {
+        discover(input, principal, caller = { kind: "unverified" }) {
+            checkHash(manifest.hash, input.expected_hash);
+            const view = projection(principal, caller);
+            const matches = searchDocuments(view.documents(), input);
+            const total = matches.length;
+            const limit = input.limit ?? 10;
+            return page({
+                hash: manifest.hash,
+                permissions: view.permissions,
+                request: {
+                    kind: "search",
+                    runtime: caller.kind === "runtime",
+                    query: normalizeSearch(input.query),
+                    filter: input.kind,
+                    entity: input.entity,
+                    mode: input.mode ?? "auto",
+                    limit,
+                },
+                after: input.after,
+                limit,
+                items: matches,
+                result: (matches, next_cursor) => ({
+                    hash: manifest.hash,
+                    kind: "search",
+                    query: input.query,
+                    total,
+                    matches,
+                    next_cursor,
+                }),
+            });
+        },
+        /**
+         * `discover` ranked by meaning: the ontology's decision model answers, for each entity the
+         * caller may see and use, whether the request concerns it. The likely ones come back most
+         * likely first, with the caller's tools and metrics for them, on one page. A model failure is
+         * thrown for the caller to fall back to `discover`; a stale `expected_hash` is refused as there.
+         */
+        async discoverByMeaning(input, principal, caller, ranking) {
+            checkHash(manifest.hash, input.expected_hash);
+            const view = projection(principal, caller);
+            const owned = new Map();
+            const withMetrics = new Map();
+            for (const tool of visibleTools(tools, principal, caller)) {
+                if ("planned" in tool.target) {
+                    const owner = tool.target.planned.name;
+                    owned.set(owner, [...(owned.get(owner) ?? []), tool.name]);
+                }
+            }
+            for (const ref of view.metrics.keys()) {
+                const name = ref.slice("metric:".length);
+                if (name.includes(".")) {
+                    const owner = name.split(".")[0];
+                    withMetrics.set(owner, [...(withMetrics.get(owner) ?? []), name]);
+                }
+            }
+            const candidates = [...view.visibleEntities.values()].filter((entity) => (input.entity === undefined || entity.name === input.entity) &&
+                (owned.has(entity.name) || withMetrics.has(entity.name)));
+            const answers = candidates.length === 0
+                ? {}
+                : await ranking.decide({
+                    model: ranking.model,
+                    state: input.query,
+                    questions: Object.fromEntries(candidates.map((entity, index) => [`entity_${index}`, concerns(entity)])),
+                    signal: AbortSignal.timeout(DECISION_TIMEOUT_MS),
+                });
+            const likely = candidates
+                .map((entity, index) => ({ entity, relevance: answers[`entity_${index}`] ?? 0 }))
+                .filter(({ relevance }) => relevance >= RELEVANT)
+                .sort((a, b) => b.relevance - a.relevance || compareRefs(a.entity.name, b.entity.name));
+            const ranked = likely.slice(0, input.limit ?? 10).map(({ entity, relevance }) => ({
+                ...summary(`entity:${entity.name}`, entity.name, entity.label, entity.description),
+                kind: "entity",
+                entity: entity.name,
+                matched_on: [],
+                relevance: Math.round(relevance * 100) / 100,
+                tools: owned.get(entity.name) ?? [],
+                metrics: withMetrics.get(entity.name) ?? [],
+            }));
+            const result = (matches) => ({
+                hash: manifest.hash,
+                kind: "search",
+                ranking: "meaning",
+                query: input.query,
+                total: likely.length,
+                matches,
+                next_cursor: null,
+            });
+            // The same response budget as a page of word matches: the likeliest that fit.
+            let fitted = result([]);
+            for (let count = 1; count <= ranked.length; count++) {
+                const next = result(ranked.slice(0, count));
+                if (discoveryResultBytes(next) > DISCOVERY_RESULT_BYTES) {
+                    if (count === 1) {
+                        throw new DiscoveryError({
+                            error: "discovery_too_large",
+                            detail: "This discovery entry exceeds the response budget.",
+                        });
+                    }
+                    break;
+                }
+                fitted = next;
+            }
+            return fitted;
+        },
+    };
+}

package/dist/service/docs.d.ts

@@ -0,0 +1,9 @@
+import { HttpRouter } from "effect/unstable/http";
+/** The API reference page: `GET /api/ontology/docs`. */
+export declare const DOCS_PATH = "/api/ontology/docs";
+/**
+ * The API reference at `/api/ontology/docs`: Scalar over the service's own `openapi.json`. The page
+ * is public and holds no part of the ontology: it loads the document as the reader, with the app's
+ * session cookie when they are signed in to the app, or with a token they paste.
+ */
+export declare const DocsRoute: import("effect/Layer").Layer<never, never, HttpRouter.HttpRouter>;

package/dist/service/docs.js

@@ -0,0 +1,197 @@
+import { Effect } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+import { BASE_PATH } from "./api.js";
+/** The API reference page: `GET /api/ontology/docs`. */
+export const DOCS_PATH = `${BASE_PATH}/docs`;
+const OPENAPI_PATH = `${BASE_PATH}/openapi.json`;
+/**
+ * Scalar's API reference, pinned with its integrity hash: a different file never runs. Bump both
+ * together: `curl -sL <url> | openssl dgst -sha384 -binary | openssl base64 -A`.
+ */
+const SCALAR_SCRIPT = {
+    src: "https://cdn.jsdelivr.net/npm/@scalar/api-reference@1.73.1/dist/browser/standalone.js",
+    integrity: "sha384-kYDGzV91Jnn3TbHINV3nt54riK2uMJDfN5Al8dAkz4FssELTBWbD8rgw32sTKfOi",
+};
+/**
+ * Scalar without its hosted extras: no telemetry, AI agent, MCP button, developer tools or fonts
+ * from scalar.com. Requests from the page go to the ontology itself, never through Scalar's proxy.
+ */
+const SCALAR_CONFIGURATION = {
+    telemetry: false,
+    agent: { disabled: true },
+    mcp: { disabled: true },
+    showDeveloperTools: "never",
+    withDefaultFonts: false,
+    hideClientButton: true,
+    documentDownloadType: "json",
+};
+/** The key the page keeps a pasted token under, for this tab only. */
+const TOKEN_STORAGE_KEY = "segmentstream-ontology-docs-token";
+/** JSON inside a `<script>`: nothing in it can close the element. */
+function scriptJson(value) {
+    return JSON.stringify(value).replace(/</g, "\\u003c");
+}
+function contentSecurityPolicy(nonce) {
+    return [
+        "default-src 'none'",
+        `script-src 'nonce-${nonce}' https://cdn.jsdelivr.net`,
+        // Scalar injects its styles at runtime.
+        "style-src 'self' 'unsafe-inline'",
+        "img-src 'self' data: blob: https:",
+        "font-src 'self' data:",
+        "connect-src 'self'",
+        "worker-src 'self' blob:",
+        "base-uri 'none'",
+        "form-action 'none'",
+        "frame-ancestors 'none'",
+    ].join("; ");
+}
+function page(nonce) {
+    const settings = scriptJson({
+        openapi: OPENAPI_PATH,
+        storageKey: TOKEN_STORAGE_KEY,
+        configuration: SCALAR_CONFIGURATION,
+    });
+    return `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="robots" content="noindex" />
+    <title>Ontology API</title>
+    <style>
+      #sign-in {
+        font: 14px/1.5 system-ui, sans-serif;
+        max-width: 28rem;
+        margin: 15vh auto;
+        padding: 0 1.5rem;
+        color: #18181b;
+      }
+      #sign-in h1 { font-size: 1.25rem; margin: 0 0 .5rem; }
+      #sign-in p { margin: 0 0 1rem; color: #52525b; }
+      #sign-in label { display: block; font-weight: 500; margin-bottom: .25rem; }
+      #sign-in input {
+        box-sizing: border-box;
+        width: 100%;
+        padding: .5rem .625rem;
+        border: 1px solid #d4d4d8;
+        border-radius: .375rem;
+        font: inherit;
+      }
+      #sign-in button {
+        margin-top: .75rem;
+        padding: .5rem .875rem;
+        border: 0;
+        border-radius: .375rem;
+        background: #18181b;
+        color: #fff;
+        font: inherit;
+        cursor: pointer;
+      }
+      #sign-in [role="alert"] { margin-top: .75rem; color: #b91c1c; }
+    </style>
+  </head>
+  <body>
+    <main id="sign-in" hidden>
+      <form>
+        <h1>Ontology API</h1>
+        <p>Sign in to the app in this browser, or paste a bearer token or an API key (ssk_…).</p>
+        <label for="token">Token or API key</label>
+        <input id="token" type="password" autocomplete="off" spellcheck="false" required />
+        <button type="submit">Open the reference</button>
+        <p role="alert"></p>
+      </form>
+    </main>
+    <div id="app"></div>
+    <script
+      nonce="${nonce}"
+      src="${SCALAR_SCRIPT.src}"
+      integrity="${SCALAR_SCRIPT.integrity}"
+      crossorigin="anonymous"
+    ></script>
+    <script nonce="${nonce}">
+      (() => {
+        const settings = ${settings};
+        const signIn = document.getElementById("sign-in");
+        const form = signIn.querySelector("form");
+        const input = document.getElementById("token");
+        const alert = signIn.querySelector("[role=alert]");
+
+        async function load(token) {
+          const response = await fetch(settings.openapi, {
+            credentials: "same-origin",
+            headers: token ? { authorization: "Bearer " + token } : {},
+          });
+
+          if (response.ok) {
+            return { spec: await response.json() };
+          }
+
+          const body = await response.json().catch(() => ({}));
+
+          return { status: response.status, detail: body.detail };
+        }
+
+        function show(spec, token) {
+          signIn.hidden = true;
+          window.Scalar.createApiReference(document.getElementById("app"), {
+            ...settings.configuration,
+            content: spec,
+            authentication: {
+              preferredSecurityScheme: "bearer",
+              ...(token ? { securitySchemes: { bearer: { token } } } : {}),
+            },
+          });
+        }
+
+        async function open(token) {
+          let result;
+
+          try {
+            result = await load(token);
+          } catch {
+            result = { status: 0 };
+          }
+
+          if (result.spec !== undefined) {
+            if (token) {
+              sessionStorage.setItem(settings.storageKey, token);
+            }
+            show(result.spec, token);
+
+            return;
+          }
+
+          sessionStorage.removeItem(settings.storageKey);
+          signIn.hidden = false;
+          alert.textContent =
+            !token && result.status === 401
+              ? ""
+              : (result.detail ?? "The reference could not be loaded. Try again shortly.");
+        }
+
+        form.addEventListener("submit", (event) => {
+          event.preventDefault();
+          open(input.value.trim());
+        });
+
+        open(sessionStorage.getItem(settings.storageKey) ?? "");
+      })();
+    </script>
+  </body>
+</html>
+`;
+}
+/**
+ * The API reference at `/api/ontology/docs`: Scalar over the service's own `openapi.json`. The page
+ * is public and holds no part of the ontology: it loads the document as the reader, with the app's
+ * session cookie when they are signed in to the app, or with a token they paste.
+ */
+export const DocsRoute = HttpRouter.add("GET", DOCS_PATH, Effect.sync(() => {
+    const nonce = crypto.randomUUID().replaceAll("-", "");
+    return HttpServerResponse.html(page(nonce)).pipe(HttpServerResponse.setHeaders({
+        "content-security-policy": contentSecurityPolicy(nonce),
+        "referrer-policy": "no-referrer",
+        "x-content-type-options": "nosniff",
+    }));
+}));