@chartcoach/catalog 0.1.7 → 0.2.0

package/src/catalog/profile.ts

@@ -0,0 +1,345 @@
+import { CatalogError } from "./errors";
+import { isJsonNumber, isJsonObject, isJsonString, type JsonObject, type JsonValue } from "./json";
+
+export type DistanceMetric = "cosine" | "l2" | "dot";
+
+export type EmbeddingBinding = Readonly<{
+  name: string;
+  model: Readonly<JsonObject>;
+  source_column: "text";
+  vector_column: "vector";
+}>;
+
+export type ProjectionMetadata = Readonly<{
+  algorithm: "empty" | "linear" | "umap";
+  options: Readonly<JsonObject>;
+}>;
+
+export type ProfileMetadata = Readonly<{
+  schema_version: 1;
+  documents_version: 1;
+  entries_digest: string;
+  manifest_digest: string;
+  embedding_functions: readonly [EmbeddingBinding];
+  dimensions: number;
+  distance_metric: DistanceMetric;
+  python_requirements: Readonly<Record<string, string>>;
+  lancedb_version: string;
+  projection: ProjectionMetadata | null;
+}>;
+
+const profileFields = [
+  "schema_version",
+  "documents_version",
+  "entries_digest",
+  "manifest_digest",
+  "embedding_functions",
+  "dimensions",
+  "distance_metric",
+  "python_requirements",
+  "lancedb_version",
+  "projection",
+] as const;
+
+const bindingFields = ["name", "model", "source_column", "vector_column"] as const;
+
+const projectionFields = ["algorithm", "options"] as const;
+
+const sha256Pattern = /^[a-f0-9]{64}$/;
+
+const distributionPattern = /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/;
+
+const versionPattern = /^[A-Za-z0-9][A-Za-z0-9.!+_-]*$/;
+
+const variablePattern = /^\$var:[^:]+$/;
+
+const sensitiveNamePattern =
+  /(?:^|[_-])(?:access[_-]?key|api[_-]?key|authorization|client[_-]?secret|cookie|credential|password|private[_-]?key|secret|signature|token)(?:$|[_-])/i;
+
+const sensitiveQueryNames = new Set([
+  "access_key",
+  "api_key",
+  "apikey",
+  "authorization",
+  "credential",
+  "password",
+  "secret",
+  "signature",
+  "sig",
+  "token",
+]);
+
+export function parseProfileMetadata(value: JsonValue): ProfileMetadata {
+  const record = requireObject(value, "Profile metadata");
+  requireExactFields(record, profileFields, "Profile metadata");
+
+  if (record.schema_version !== 1) {
+    throw new CatalogError("Profile schema_version must be 1.");
+  }
+
+  if (record.documents_version !== 1) {
+    throw new CatalogError("Profile documents_version must be 1.");
+  }
+
+  const entriesDigest = requireSha256(record.entries_digest, "Profile entries_digest");
+  const manifestDigest = requireSha256(record.manifest_digest, "Profile manifest_digest");
+
+  if (!Array.isArray(record.embedding_functions) || record.embedding_functions.length !== 1) {
+    throw new CatalogError("Profile must contain exactly one embedding binding.");
+  }
+
+  const binding = parseBinding(record.embedding_functions[0]!);
+  const dimensions = record.dimensions;
+
+  if (!isJsonNumber(dimensions) || !Number.isSafeInteger(dimensions) || dimensions < 1) {
+    throw new CatalogError("Profile dimensions must be a positive safe integer.");
+  }
+
+  const distanceMetric = record.distance_metric;
+
+  if (distanceMetric !== "cosine" && distanceMetric !== "l2" && distanceMetric !== "dot") {
+    throw new CatalogError("Profile distance_metric must be cosine, l2, or dot.");
+  }
+
+  const pythonRequirements = parseRequirements(record.python_requirements);
+  const lancedbVersion = requireVersion(record.lancedb_version, "Profile lancedb_version");
+  const projection = record.projection === null ? null : parseProjection(record.projection);
+
+  return Object.freeze({
+    schema_version: 1,
+    documents_version: 1,
+    entries_digest: entriesDigest,
+    manifest_digest: manifestDigest,
+    embedding_functions: Object.freeze([binding] as const),
+    dimensions,
+    distance_metric: distanceMetric,
+    python_requirements: pythonRequirements,
+    lancedb_version: lancedbVersion,
+    projection,
+  });
+}
+
+function parseBinding(value: JsonValue): EmbeddingBinding {
+  const record = requireObject(value, "Profile embedding binding");
+  requireExactFields(record, bindingFields, "Profile embedding binding");
+  const name = record.name;
+
+  if (!isJsonString(name) || !name || name !== name.trim()) {
+    throw new CatalogError("Profile embedding name must be a non-empty string.");
+  }
+
+  if (record.source_column !== "text" || record.vector_column !== "vector") {
+    throw new CatalogError("Profile embedding binding must map text to vector.");
+  }
+
+  const model = requireObject(record.model, "Profile embedding model");
+  validateModel(model);
+
+  return Object.freeze({
+    name,
+    model: freezeJsonObject(model),
+    source_column: "text",
+    vector_column: "vector",
+  });
+}
+
+function parseProjection(value: JsonValue): ProjectionMetadata {
+  const record = requireObject(value, "Profile projection");
+  requireExactFields(record, projectionFields, "Profile projection");
+  const algorithm = record.algorithm;
+
+  if (algorithm !== "empty" && algorithm !== "linear" && algorithm !== "umap") {
+    throw new CatalogError("Profile projection algorithm is unsupported.");
+  }
+
+  return Object.freeze({
+    algorithm,
+    options: freezeJsonObject(requireObject(record.options, "Profile projection options")),
+  });
+}
+
+function parseRequirements(value: JsonValue | undefined): Readonly<Record<string, string>> {
+  const record = requireObject(value, "Profile Python requirements");
+  const requirements: Record<string, string> = Object.create(null);
+
+  for (const [name, requirement] of Object.entries(record)) {
+    if (
+      !distributionPattern.test(name) ||
+      !isJsonString(requirement) ||
+      !versionPattern.test(requirement) ||
+      !/[0-9]/.test(requirement)
+    ) {
+      throw new CatalogError(
+        "Profile python_requirements must map distribution names to exact versions.",
+      );
+    }
+
+    requirements[name] = requirement;
+  }
+
+  return Object.freeze(requirements);
+}
+
+function validateModel(model: JsonObject): void {
+  for (const [key, value] of Object.entries(model)) {
+    const variables = variableValues(value);
+
+    if (
+      variables.some((variable) => !variablePattern.test(variable)) ||
+      (variables.length > 0 && !isJsonString(value))
+    ) {
+      throw new CatalogError(
+        `Profile embedding setting ${JSON.stringify(key)} has an invalid variable.`,
+      );
+    }
+
+    if (
+      sensitiveNamePattern.test(key) &&
+      value !== null &&
+      (!isJsonString(value) || !variablePattern.test(value))
+    ) {
+      throw new CatalogError(
+        `Profile embedding setting ${JSON.stringify(key)} must use a registry variable.`,
+      );
+    }
+
+    if (
+      key === "default_headers" &&
+      value !== null &&
+      (!isJsonObject(value) || Object.keys(value).length > 0)
+    ) {
+      throw new CatalogError("Profile embedding default_headers must be empty.");
+    }
+
+    if (containsSensitiveKey(value)) {
+      throw new CatalogError(
+        `Profile embedding setting ${JSON.stringify(key)} contains authentication data.`,
+      );
+    }
+
+    rejectCredentialUrls(value, key);
+  }
+
+  if (Object.hasOwn(model, "trust_remote_code") && model.trust_remote_code !== false) {
+    throw new CatalogError("Profile embedding must disable remote model code.");
+  }
+}
+
+function containsSensitiveKey(value: JsonValue): boolean {
+  if (Array.isArray(value)) return value.some(containsSensitiveKey);
+
+  if (!isJsonObject(value)) return false;
+
+  return Object.entries(value).some(
+    ([key, item]) => sensitiveNamePattern.test(key) || containsSensitiveKey(item),
+  );
+}
+
+function rejectCredentialUrls(value: JsonValue, path: string): void {
+  if (Array.isArray(value)) {
+    value.forEach((item, index) => rejectCredentialUrls(item, `${path}[${index}]`));
+
+    return;
+  }
+
+  if (isJsonObject(value)) {
+    for (const [key, item] of Object.entries(value)) {
+      rejectCredentialUrls(item, `${path}.${key}`);
+    }
+
+    return;
+  }
+
+  if (!isJsonString(value)) return;
+  let parsed: URL;
+
+  try {
+    parsed = new URL(value);
+  } catch {
+    return;
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return;
+
+  if (parsed.username || parsed.password) {
+    throw new CatalogError(
+      `Profile embedding URL setting ${JSON.stringify(path)} contains credentials.`,
+    );
+  }
+
+  if (
+    Array.from(parsed.searchParams.keys()).some(
+      (name) => sensitiveQueryNames.has(name.toLowerCase()) || sensitiveNamePattern.test(name),
+    )
+  ) {
+    throw new CatalogError(
+      `Profile embedding URL setting ${JSON.stringify(path)} contains authentication parameters.`,
+    );
+  }
+}
+
+function variableValues(value: JsonValue): string[] {
+  if (isJsonString(value)) return value.startsWith("$var:") ? [value] : [];
+
+  if (Array.isArray(value)) return value.flatMap(variableValues);
+
+  if (!isJsonObject(value)) return [];
+
+  return Object.values(value).flatMap(variableValues);
+}
+
+function freezeJsonObject(value: JsonObject): Readonly<JsonObject> {
+  const result: JsonObject = Object.create(null);
+
+  for (const [key, item] of Object.entries(value)) {
+    result[key] = freezeJson(item);
+  }
+
+  return Object.freeze(result);
+}
+
+function freezeJson(value: JsonValue): JsonValue {
+  if (Object.prototype.toString.call(value) === "[object Number]" && !isJsonNumber(value)) {
+    throw new CatalogError("Profile metadata must contain finite JSON numbers.");
+  }
+
+  if (Array.isArray(value)) return Object.freeze(value.map(freezeJson));
+
+  if (isJsonObject(value)) return freezeJsonObject(value);
+
+  return value;
+}
+
+function requireObject(value: JsonValue | undefined, label: string): JsonObject {
+  if (!isJsonObject(value)) throw new CatalogError(`${label} must be a JSON object.`);
+
+  return value;
+}
+
+function requireExactFields(value: JsonObject, fields: readonly string[], label: string): void {
+  const actual = Object.keys(value);
+  const missing = fields.filter((field) => !Object.hasOwn(value, field));
+  const unknown = actual.filter((field) => !fields.includes(field));
+
+  if (missing.length > 0)
+    throw new CatalogError(`${label} is missing fields: ${missing.join(", ")}.`);
+
+  if (unknown.length > 0)
+    throw new CatalogError(`${label} has unsupported fields: ${unknown.join(", ")}.`);
+}
+
+function requireSha256(value: JsonValue | undefined, label: string): string {
+  if (!isJsonString(value) || !sha256Pattern.test(value)) {
+    throw new CatalogError(`${label} must be a lowercase SHA-256 digest.`);
+  }
+
+  return value;
+}
+
+function requireVersion(value: JsonValue | undefined, label: string): string {
+  if (!isJsonString(value) || !versionPattern.test(value) || !/[0-9]/.test(value)) {
+    throw new CatalogError(`${label} must be a non-empty exact version.`);
+  }
+
+  return value;
+}

package/src/catalog/query.ts

@@ -0,0 +1,125 @@
+import { CatalogError } from "./errors";
+import { isJsonString } from "./json";
+import type { Catalog, Guideline } from "./model";
+
+export type QueryOptions = Readonly<{
+  ids?: readonly string[];
+  labels?: readonly string[];
+  labelPrefixes?: readonly string[];
+  contains?: string;
+  limit?: number;
+}>;
+
+export type EntryCandidate = Readonly<{
+  id: string;
+  title: string;
+  description: string;
+  labels: readonly string[];
+}>;
+
+export function queryCatalog(
+  catalog: Catalog,
+  options: QueryOptions = {},
+): readonly EntryCandidate[] {
+  if (Object.prototype.toString.call(options) !== "[object Object]") {
+    throw new CatalogError("Query options must be an object.");
+  }
+
+  const ids = stringArray(options.ids, "ids");
+  const labels = stringArray(options.labels, "labels");
+  const labelPrefixes = stringArray(options.labelPrefixes, "labelPrefixes");
+  const limit = options.limit ?? 50;
+
+  if (!Number.isSafeInteger(limit) || limit < 1) {
+    throw new CatalogError("Query limit must be at least 1.");
+  }
+
+  validateLabels(catalog, labels);
+  validateLabelPrefixes(catalog, labelPrefixes);
+  const selected = ids.length > 0 ? ids.map((id) => catalog.require(id)) : [...catalog.guidelines];
+  let contains: string | undefined;
+
+  if (options.contains !== undefined) {
+    if (!isJsonString(options.contains)) {
+      throw new CatalogError("Query contains must be a string.");
+    }
+
+    if (options.contains.length > 0) {
+      contains = normalizeSearchText(options.contains);
+
+      if (!contains) {
+        throw new CatalogError(
+          "contains must not consist only of whitespace, hyphens, or underscores.",
+        );
+      }
+    }
+  }
+
+  return Object.freeze(
+    selected
+      .filter((guideline) => labels.every((label) => guideline.labels.includes(label)))
+      .filter((guideline) =>
+        labelPrefixes.every((prefix) => guideline.labels.some((label) => label.startsWith(prefix))),
+      )
+      .filter((guideline) => contains === undefined || matchesText(guideline, contains))
+      .slice(0, limit)
+      .map((guideline) =>
+        Object.freeze({
+          id: guideline.id,
+          title: guideline.title,
+          description: guideline.description,
+          labels: Object.freeze([...guideline.labels]),
+        }),
+      ),
+  );
+}
+
+function validateLabels(catalog: Catalog, labels: readonly string[]): void {
+  const available = new Set(catalog.labels());
+  const missing = Array.from(new Set(labels.filter((label) => !available.has(label)))).sort();
+
+  if (missing.length === 0) return;
+  throw new CatalogError(`Unknown label(s): ${missing.join(", ")}`, {
+    code: "lookup",
+    details: { labels: missing },
+    hints: ["Call `await catalog.describe()` to inspect label families."],
+  });
+}
+
+function validateLabelPrefixes(catalog: Catalog, prefixes: readonly string[]): void {
+  const available = catalog.labels();
+
+  const missing = Array.from(
+    new Set(prefixes.filter((prefix) => !available.some((label) => label.startsWith(prefix)))),
+  ).sort();
+
+  if (missing.length === 0) return;
+  throw new CatalogError(`No labels match prefix(es): ${missing.join(", ")}`, {
+    code: "lookup",
+    details: { label_prefixes: missing },
+    hints: ["Call `await catalog.describe()` to inspect label families."],
+  });
+}
+
+function stringArray(value: readonly string[] | undefined, name: string): readonly string[] {
+  if (value === undefined) return Object.freeze([]);
+
+  if (!Array.isArray(value) || !value.every(isJsonString)) {
+    throw new CatalogError(`${name} must be an array of strings.`);
+  }
+
+  return value;
+}
+
+function matchesText(guideline: Guideline, contains: string): boolean {
+  return [guideline.id, guideline.title, guideline.description].some((value) =>
+    normalizeSearchText(value).includes(contains),
+  );
+}
+
+function normalizeSearchText(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[-_\s]+/g, " ")
+    .trim();
+}

package/src/catalog/read.ts

@@ -0,0 +1,101 @@
+import { CatalogError } from "./errors";
+import { isJsonString } from "./json";
+import type { Catalog, GuidelineSection } from "./model";
+import { sourceRecords, type FullSourceRecord, type MinimalSourceRecord } from "./references";
+
+export type SourceDetail = "none" | "minimal" | "full";
+
+export type ReadOptions = Readonly<{
+  ids: readonly string[];
+  roles?: readonly string[];
+  sourceDetail?: SourceDetail;
+}>;
+
+export type GuidelineEntryRecord = Readonly<{
+  id: string;
+  title: string;
+  description: string;
+  labels: readonly string[];
+  sections: readonly GuidelineSection[];
+  sources: readonly (MinimalSourceRecord | FullSourceRecord)[];
+  references?: readonly string[];
+}>;
+
+export function readCatalog(
+  catalog: Catalog,
+  options: ReadOptions,
+): readonly GuidelineEntryRecord[] {
+  if (Object.prototype.toString.call(options) !== "[object Object]") {
+    throw new CatalogError("Read options must be an object.");
+  }
+
+  if (!Array.isArray(options.ids) || !options.ids.every(isJsonString)) {
+    throw new CatalogError("ids must be an array of strings.");
+  }
+
+  const roles = options.roles ?? [];
+
+  if (!Array.isArray(roles) || !roles.every(isJsonString)) {
+    throw new CatalogError("roles must be an array of strings.");
+  }
+
+  const sourceDetail = options.sourceDetail ?? "minimal";
+
+  if (sourceDetail !== "none" && sourceDetail !== "minimal" && sourceDetail !== "full") {
+    throw new CatalogError(`Unknown source detail: ${JSON.stringify(sourceDetail)}.`, {
+      hints: ["Choose one of: none, minimal, full."],
+    });
+  }
+
+  const missingRoles = Array.from(
+    new Set(roles.filter((role) => !Object.hasOwn(catalog.manifest.sectionRoles, role))),
+  ).sort();
+
+  if (missingRoles.length > 0) {
+    throw new CatalogError(`Unknown section role(s): ${missingRoles.join(", ")}`, {
+      code: "lookup",
+      details: { roles: missingRoles },
+      hints: ["Call `await catalog.describe()` to inspect section roles."],
+    });
+  }
+
+  if (options.ids.length === 0) return Object.freeze([]);
+
+  const roleSet = roles.length > 0 ? new Set(roles) : undefined;
+
+  return Object.freeze(
+    options.ids.map((id) => {
+      const guideline = catalog.require(id);
+
+      const sections = Object.freeze(
+        guideline.sections
+          .filter((section) => roleSet === undefined || roleSet.has(section.role))
+          .map((section) =>
+            Object.freeze({
+              role: section.role,
+              title: section.title,
+              content: section.content,
+            }),
+          ),
+      );
+
+      const record = {
+        id: guideline.id,
+        title: guideline.title,
+        description: guideline.description,
+        labels: Object.freeze([...guideline.labels]),
+        sections,
+        sources:
+          sourceDetail === "none"
+            ? Object.freeze([])
+            : sourceRecords(catalog, guideline.id, sourceDetail),
+      };
+
+      return Object.freeze(
+        sourceDetail === "full"
+          ? { ...record, references: Object.freeze([...guideline.references]) }
+          : record,
+      );
+    }),
+  );
+}