@chartcoach/catalog 0.1.6 → 0.2.0

package/src/catalog/artifacts.ts

@@ -1,122 +1,331 @@
-import { CHARTCOACH_DEFAULTS } from "./chartcoach-defaults";
 import { CatalogError } from "./errors";
-export { CHARTCOACH_DEFAULTS, type ChartCoachDefaults } from "./chartcoach-defaults";
-
-const defaultCatalogReleaseRootUrl = new URL(
-  `catalog/releases/${CHARTCOACH_DEFAULTS.catalogVersion}/${CHARTCOACH_DEFAULTS.catalogDigest}/`,
-  `${CHARTCOACH_DEFAULTS.catalogArtifactBaseUrl}/`,
-).toString();
-
-export const DEFAULT_CATALOG = {
-  version: CHARTCOACH_DEFAULTS.catalogVersion,
-  digest: CHARTCOACH_DEFAULTS.catalogDigest,
-  releaseRootUrl: defaultCatalogReleaseRootUrl,
-  metadataUrl: new URL("metadata.json", defaultCatalogReleaseRootUrl).toString(),
-  entriesUrl: new URL("entries.parquet", defaultCatalogReleaseRootUrl).toString(),
-  manifestUrl: new URL("MANIFEST.md", defaultCatalogReleaseRootUrl).toString(),
-} as const;
-
-export type ArtifactKind = "manifest" | "entries" | "lancedb-index";
-
-export type ArtifactDescriptor = {
-  kind: ArtifactKind;
-  path: string;
-  digest: string;
+import { canonicalJson, compareUnicode, sha256Bytes } from "./identity";
+import { isJsonNumber, isJsonObject, isJsonString, type JsonObject, type JsonValue } from "./json";
+
+const CATALOG_ARTIFACT_BASE_URL = "https://artifacts.chartcoach.dev/";
+
+const RELEASE_SCHEMA_VERSION = 1;
+
+const REQUIRED_ARTIFACTS = ["MANIFEST.md", "entries.parquet"] as const;
+
+const FORBIDDEN_PATH_CHARACTERS = new Set('<>:"\\|?*#%');
+
+const WINDOWS_DEVICE_NAMES = new Set(["CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"]);
+
+export type ReleaseArtifact = Readonly<{
+  sha256: string;
   bytes: number;
-  format?: string;
-  rows?: number;
-  [key: string]: unknown;
-};
+}>;
 
-export type CatalogReleaseMetadata = {
-  version: string;
+export type CatalogRelease = Readonly<{
+  schema_version: 1;
   digest: string;
-  artifacts: ArtifactDescriptor[];
-};
-
-export function parseCatalogReleaseMetadata(value: unknown): CatalogReleaseMetadata {
-  if (!isRecord(value)) {
-    throw new CatalogError("Catalog release metadata must be an object.");
-  }
-  const artifacts = value.artifacts;
-  if (!Array.isArray(artifacts)) {
-    throw new CatalogError("Catalog release metadata artifacts must be a list.");
-  }
-  const metadata = {
-    version: requiredString(value, "version"),
-    digest: requiredString(value, "digest"),
-    artifacts: artifacts.map(parseArtifactDescriptor),
-  };
-  catalogArtifact(metadata, "manifest");
-  catalogArtifact(metadata, "entries");
-  return metadata;
-}
-
-export function catalogArtifactUrl(baseUrl: string | URL, descriptor: ArtifactDescriptor): string {
-  return new URL(descriptor.path, baseUrl).toString();
-}
-
-export function catalogArtifact(
-  metadata: CatalogReleaseMetadata,
-  kind: ArtifactKind,
-): ArtifactDescriptor {
-  const descriptor = metadata.artifacts.find((item) => item.kind === kind);
-  if (!descriptor) {
-    throw new CatalogError(`Catalog release metadata is missing a ${kind} artifact.`);
-  }
-  return descriptor;
-}
-
-function parseArtifactDescriptor(value: unknown): ArtifactDescriptor {
-  if (!isRecord(value)) {
-    throw new CatalogError("Catalog artifact descriptor must be an object.");
-  }
-  const kind = value.kind;
-  if (kind !== "manifest" && kind !== "entries" && kind !== "lancedb-index") {
-    throw new CatalogError(`Unsupported catalog artifact kind: ${String(kind)}`);
-  }
-  const path = requiredString(value, "path");
-  validateRelativePath(path);
-  const descriptor: ArtifactDescriptor = {
-    kind,
-    path,
-    digest: requiredString(value, "digest"),
-    bytes: requiredInteger(value, "bytes"),
-  };
-  if (typeof value.format === "string") descriptor.format = value.format;
-  if (typeof value.rows === "number" && Number.isInteger(value.rows)) {
-    descriptor.rows = value.rows;
-  }
-  for (const [key, rawValue] of Object.entries(value)) {
-    if (!["kind", "path", "digest", "bytes", "format", "rows"].includes(key)) {
-      descriptor[key] = rawValue;
+  artifacts: Readonly<Record<string, ReleaseArtifact>>;
+}>;
+
+export function parseCatalogRelease(value: JsonValue): CatalogRelease {
+  const record = requireRecord(value, "Catalog release");
+  requireExactKeys(record, ["schema_version", "digest", "artifacts"], "Catalog release");
+  requireSchemaVersion(record);
+  const artifactRecords = requireRecord(record.artifacts, "Catalog artifacts");
+  const artifacts: Record<string, ReleaseArtifact> = Object.create(null);
+
+  for (const [path, artifact] of Object.entries(artifactRecords)
+    .map(([path, value]): [string, ReleaseArtifact] => [
+      requireRelativePath(path, "Catalog artifact path"),
+      parseReleaseArtifact(value),
+    ])
+    .sort(([left], [right]) => compareUnicode(left, right))) {
+    artifacts[path] = artifact;
+  }
+
+  validateArtifactPaths(Object.keys(artifacts));
+
+  for (const path of REQUIRED_ARTIFACTS) {
+    if (!Object.hasOwn(artifacts, path)) {
+      throw new CatalogError(`Catalog release is missing artifact: ${path}.`);
+    }
+  }
+
+  return Object.freeze({
+    schema_version: RELEASE_SCHEMA_VERSION,
+    digest: requiredSha256(record, "digest", "Catalog release"),
+    artifacts: Object.freeze(artifacts),
+  });
+}
+
+export function copyCatalogRelease(release: CatalogRelease): CatalogRelease {
+  return parseCatalogRelease({
+    schema_version: release.schema_version,
+    digest: release.digest,
+    artifacts: Object.fromEntries(
+      Object.entries(release.artifacts).map(([path, artifact]) => [
+        path,
+        { sha256: artifact.sha256, bytes: artifact.bytes },
+      ]),
+    ),
+  });
+}
+
+export async function assertReleaseDigest(
+  release: CatalogRelease,
+  expected = release.digest,
+  label = "Catalog release",
+): Promise<void> {
+  const actual = await catalogReleaseDigest(release);
+
+  if (release.digest !== actual) {
+    throw new CatalogError(`${label} digest ${release.digest} does not match its artifact set.`, {
+      code: "integrity",
+      details: { expected_digest: release.digest, actual_digest: actual },
+    });
+  }
+
+  if (release.digest !== expected) {
+    throw new CatalogError(`${label} digest ${release.digest} does not match ${expected}.`, {
+      code: "integrity",
+      details: { expected_digest: expected, actual_digest: release.digest },
+    });
+  }
+}
+
+export async function assertReleaseArtifactBytes(
+  path: string,
+  artifact: ReleaseArtifact,
+  data: ArrayBuffer | ArrayBufferView,
+): Promise<void> {
+  const bytes = bytesView(data);
+
+  if (bytes.byteLength !== artifact.bytes) {
+    throw new CatalogError(`Catalog artifact byte count mismatch: ${path}`, {
+      code: "integrity",
+      details: { path, expected_bytes: artifact.bytes, actual_bytes: bytes.byteLength },
+    });
+  }
+
+  const actual = await sha256Bytes(bytes);
+
+  if (actual !== artifact.sha256) {
+    throw new CatalogError(`Catalog artifact SHA-256 mismatch: ${path}`, {
+      code: "integrity",
+      details: { path, expected_sha256: artifact.sha256, actual_sha256: actual },
+    });
+  }
+}
+
+export function catalogUrl(): string {
+  return new URL("catalog.json", CATALOG_ARTIFACT_BASE_URL).toString();
+}
+
+export function releaseArtifact(release: CatalogRelease, path: string): ReleaseArtifact {
+  const artifact = release.artifacts[path];
+
+  if (artifact === undefined) {
+    throw new CatalogError(`Catalog release is missing artifact: ${path}.`, {
+      code: "integrity",
+      details: { path },
+    });
+  }
+
+  return artifact;
+}
+
+export function releaseArtifactUrl(releaseUrl: string | URL, path: string): string {
+  return new URL(path, releaseUrl).toString();
+}
+
+export function assertReleaseLocation(releaseUrl: string | URL, release: CatalogRelease): void {
+  const url = requireReleaseUrl(releaseUrl);
+
+  if (url.pathname.split("/").at(-1) !== "release.json") return;
+  const parent = url.pathname.split("/").at(-2);
+
+  if (parent && /^[a-f0-9]{64}$/.test(parent) && parent !== release.digest) {
+    throw new CatalogError("Catalog release digest does not match its digest-addressed location.", {
+      code: "integrity",
+      details: { expected_digest: parent, actual_digest: release.digest },
+    });
+  }
+}
+
+export function sanitizeReleaseUrl(value: string | URL): string {
+  const url = requireReleaseUrl(value);
+  url.username = "";
+  url.password = "";
+  url.search = "";
+  url.hash = "";
+
+  return url.toString();
+}
+
+function requireReleaseUrl(value: string | URL): URL {
+  try {
+    return new URL(value);
+  } catch {
+    throw new CatalogError("Catalog release URL must be absolute.");
+  }
+}
+
+function parseReleaseArtifact(value: JsonValue): ReleaseArtifact {
+  const record = requireRecord(value, "Catalog release artifact");
+  requireExactKeys(record, ["sha256", "bytes"], "Catalog release artifact");
+
+  return Object.freeze({
+    sha256: requiredSha256(record, "sha256", "Catalog release artifact"),
+    bytes: requiredInteger(record, "bytes", "Catalog release artifact"),
+  });
+}
+
+function validateArtifactPaths(paths: string[]): void {
+  const seen = new Map<string, string>();
+
+  for (const path of paths) {
+    const folded = asciiLower(path);
+    const previous = seen.get(folded);
+
+    if (previous !== undefined) {
+      throw new CatalogError(`Catalog artifact paths collide: ${previous}, ${path}.`);
+    }
+
+    seen.set(folded, path);
+  }
+
+  for (const [folded, path] of seen) {
+    const parts = folded.split("/");
+
+    if (folded === "release.json") {
+      throw new CatalogError(`Catalog artifact path is reserved: ${path}.`);
+    }
+
+    for (let length = 1; length < parts.length; length += 1) {
+      const ancestor = parts.slice(0, length).join("/");
+      const previous = seen.get(ancestor);
+
+      if (previous !== undefined) {
+        throw new CatalogError(`Catalog artifact paths collide: ${previous}, ${path}.`);
+      }
     }
   }
-  return descriptor;
 }
 
-function validateRelativePath(path: string): void {
-  if (path.startsWith("/") || path.split("/").includes("..")) {
-    throw new CatalogError(`Catalog artifact path must be relative: ${path}`);
+async function catalogReleaseDigest(release: CatalogRelease): Promise<string> {
+  return sha256Bytes(
+    new TextEncoder().encode(
+      canonicalJson({
+        schema_version: release.schema_version,
+        artifacts: release.artifacts,
+      }),
+    ),
+  );
+}
+
+function bytesView(data: ArrayBuffer | ArrayBufferView): Uint8Array {
+  if (data instanceof ArrayBuffer) return new Uint8Array(data);
+
+  return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+}
+
+function requireRecord(value: JsonValue, label: string): JsonObject {
+  if (!isJsonObject(value)) throw new CatalogError(`${label} must be an object.`);
+
+  return value;
+}
+
+function requireExactKeys(value: JsonObject, expected: readonly string[], label: string): void {
+  const actual = Object.keys(value);
+  const missing = expected.filter((key) => !actual.includes(key)).sort();
+  const unexpected = actual.filter((key) => !expected.includes(key)).sort();
+
+  if (missing.length > 0) {
+    throw new CatalogError(`${label} is missing fields: ${missing.join(", ")}.`);
+  }
+
+  if (unexpected.length > 0) {
+    throw new CatalogError(`${label} has unsupported fields: ${unexpected.join(", ")}.`);
+  }
+}
+
+function requireSchemaVersion(value: JsonObject): void {
+  const version = requiredInteger(value, "schema_version", "Catalog release");
+
+  if (version !== RELEASE_SCHEMA_VERSION) {
+    throw new CatalogError(`Catalog release schema_version must be ${RELEASE_SCHEMA_VERSION}.`);
   }
 }
 
-function requiredString(value: Record<string, unknown>, key: string): string {
+function requireRelativePath(path: string, label: string): string {
+  const parts = path.split("/");
+
+  if (!pathIsAscii(path) || parts.some((part) => !isPortablePathSegment(part))) {
+    throw new CatalogError(`${label} must be relative: ${path}`);
+  }
+
+  return path;
+}
+
+function isPortablePathSegment(value: string): boolean {
+  if (
+    value === "" ||
+    value === "." ||
+    value === ".." ||
+    value.endsWith(".") ||
+    Array.from(value).some((character) => FORBIDDEN_PATH_CHARACTERS.has(character))
+  ) {
+    return false;
+  }
+
+  const basename = value.split(".", 1)[0]!.toUpperCase();
+
+  return !WINDOWS_DEVICE_NAMES.has(basename) && !/^(?:COM|LPT)[1-9]$/.test(basename);
+}
+
+function requiredString(value: JsonObject, key: string, label: string): string {
   const raw = value[key];
-  if (typeof raw !== "string" || raw.length === 0) {
-    throw new CatalogError(`Catalog release metadata ${key} must be a string.`);
+
+  if (!isJsonString(raw) || raw.length === 0) {
+    throw new CatalogError(`${label} ${key} must be a string.`);
   }
+
   return raw;
 }
 
-function requiredInteger(value: Record<string, unknown>, key: string): number {
+function requiredInteger(value: JsonObject, key: string, label: string): number {
   const raw = value[key];
-  if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 0) {
-    throw new CatalogError(`Catalog release metadata ${key} must be a non-negative integer.`);
+
+  if (!isJsonNumber(raw) || !Number.isSafeInteger(raw) || raw < 0) {
+    throw new CatalogError(
+      `${label} ${key} must be an integer from 0 through ${Number.MAX_SAFE_INTEGER}.`,
+    );
   }
+
   return raw;
 }
 
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return Boolean(value && typeof value === "object" && !Array.isArray(value));
+function requiredSha256(value: JsonObject, key: string, label: string): string {
+  const digest = requiredString(value, key, label);
+  requireSha256(digest, `${label} ${key}`);
+
+  return digest;
+}
+
+function requireSha256(value: string, label: string): void {
+  if (!/^[a-f0-9]{64}$/.test(value)) {
+    throw new CatalogError(`${label} must be a lowercase SHA-256 digest.`);
+  }
+}
+
+function pathIsAscii(value: string): boolean {
+  if (value.length === 0) return false;
+
+  for (const character of value) {
+    const code = character.charCodeAt(0);
+
+    if (code < 0x21 || code > 0x7e) return false;
+  }
+
+  return true;
+}
+
+function asciiLower(value: string): string {
+  return value.replace(/[A-Z]/g, (character) => character.toLowerCase());
 }

package/src/catalog/description.ts

@@ -0,0 +1,198 @@
+import { CatalogError } from "./errors";
+import { catalogEntriesDigest, manifestDigest } from "./identity";
+import type { Catalog } from "./model";
+import type { ProfileMetadata } from "./profile";
+import type { ReleaseProfile } from "./profile-layout";
+import type { ManifestDefinition } from "./manifest";
+import { catalogTableInfo, type TableInfo } from "./tables";
+
+export type DescribeOptions = Readonly<{
+  profile?: string;
+  signal?: AbortSignal;
+}>;
+
+export type ProfileInfo = Readonly<{
+  name: string;
+  profile_schema_version: 1;
+  documents_version: 1;
+  embedding_functions: ProfileMetadata["embedding_functions"];
+  dimensions: number;
+  distance_metric: ProfileMetadata["distance_metric"];
+  python_requirements: ProfileMetadata["python_requirements"];
+  lancedb_version: string;
+  projection: ProfileMetadata["projection"];
+}>;
+
+export type CatalogInfo = Readonly<{
+  resolved_location: string | null;
+  release_digest: string | null;
+  entries_digest: string;
+  manifest_digest: string;
+  tables: readonly TableInfo[];
+  section_roles: readonly ManifestDefinition[];
+  label_families: readonly ManifestDefinition[];
+  profiles: readonly string[];
+  profile: ProfileInfo | null;
+}>;
+
+export type ProfileLoader = (
+  profile: ReleaseProfile,
+  signal?: AbortSignal,
+) => Promise<ProfileMetadata>;
+
+export type DescriptionContext = Readonly<{
+  resolvedLocation: string | null;
+  releaseDigest: string | null;
+  profiles: readonly ReleaseProfile[];
+  profileLoader?: ProfileLoader;
+  profileCache: Map<string, ProfileMetadata>;
+}>;
+
+type CatalogIdentity = Readonly<{
+  entriesDigest: string;
+  manifestDigest: string;
+}>;
+
+const identityCache = new WeakMap<Catalog, CatalogIdentity>();
+
+export async function describeCatalog(
+  catalog: Catalog,
+  context: DescriptionContext,
+  options: DescribeOptions = {},
+): Promise<CatalogInfo> {
+  if (Object.prototype.toString.call(options) !== "[object Object]") {
+    throw new CatalogError("Description options must be an object.");
+  }
+
+  const identity = await catalogIdentity(catalog);
+  const selected = selectProfile(context.profiles, options.profile);
+  let profile: ProfileInfo | null = null;
+
+  if (selected) {
+    options.signal?.throwIfAborted();
+    const metadata = await loadProfile(context, selected, options.signal);
+
+    if (metadata.entries_digest !== identity.entriesDigest) {
+      throw new CatalogError("Profile entries digest does not match the catalog entries.", {
+        code: "incompatible_profile",
+        details: { profile: selected.name },
+      });
+    }
+
+    if (metadata.manifest_digest !== identity.manifestDigest) {
+      throw new CatalogError("Profile manifest digest does not match MANIFEST.md.", {
+        code: "incompatible_profile",
+        details: { profile: selected.name },
+      });
+    }
+
+    if ((metadata.projection === null) !== (selected.projection === null)) {
+      throw new CatalogError("Profile projection metadata does not match the release inventory.", {
+        code: "incompatible_profile",
+        details: { profile: selected.name },
+      });
+    }
+
+    context.profileCache.set(selected.name, metadata);
+    profile = profileInfo(selected.name, metadata);
+  }
+
+  return Object.freeze({
+    resolved_location: context.resolvedLocation,
+    release_digest: context.releaseDigest,
+    entries_digest: identity.entriesDigest,
+    manifest_digest: identity.manifestDigest,
+    tables: catalogTableInfo(catalog),
+    section_roles: vocabulary(Object.values(catalog.manifest.sectionRoles)),
+    label_families: vocabulary(Object.values(catalog.manifest.labelFamilies)),
+    profiles: Object.freeze(context.profiles.map((item) => item.name)),
+    profile,
+  });
+}
+
+async function catalogIdentity(catalog: Catalog): Promise<CatalogIdentity> {
+  const cached = identityCache.get(catalog);
+
+  if (cached) return cached;
+
+  const [entriesDigest, catalogManifestDigest] = await Promise.all([
+    catalogEntriesDigest(catalog.guidelines),
+    manifestDigest(catalog.manifest.markdown),
+  ]);
+
+  const identity = Object.freeze({ entriesDigest, manifestDigest: catalogManifestDigest });
+  identityCache.set(catalog, identity);
+
+  return identity;
+}
+
+function selectProfile(
+  profiles: readonly ReleaseProfile[],
+  name: string | undefined,
+): ReleaseProfile | undefined {
+  if (name === undefined) return undefined;
+
+  if (!name) throw new CatalogError("Profile must be a non-empty string.");
+  const profile = profiles.find((item) => item.name === name);
+
+  if (profile) return profile;
+  throw new CatalogError(`Unknown profile: ${name}`, {
+    code: "lookup",
+    details: { profile: name, available: profiles.map((item) => item.name) },
+    hints: [
+      `Available profiles: ${profiles.map((item) => JSON.stringify(item.name)).join(", ") || "none"}.`,
+    ],
+  });
+}
+
+async function loadProfile(
+  context: DescriptionContext,
+  profile: ReleaseProfile,
+  signal?: AbortSignal,
+): Promise<ProfileMetadata> {
+  const cached = context.profileCache.get(profile.name);
+
+  if (cached) return cached;
+
+  if (!context.profileLoader) {
+    throw new CatalogError("Profile metadata loading is unavailable for caller-provided bytes.", {
+      code: "unavailable_capability",
+      details: { profile: profile.name },
+      hints: ["Use openCatalog with an HTTP or HTTPS release URL."],
+    });
+  }
+
+  return context.profileLoader(profile, signal);
+}
+
+function profileInfo(name: string, metadata: ProfileMetadata): ProfileInfo {
+  return Object.freeze({
+    name,
+    profile_schema_version: metadata.schema_version,
+    documents_version: metadata.documents_version,
+    embedding_functions: metadata.embedding_functions,
+    dimensions: metadata.dimensions,
+    distance_metric: metadata.distance_metric,
+    python_requirements: metadata.python_requirements,
+    lancedb_version: metadata.lancedb_version,
+    projection: metadata.projection,
+  });
+}
+
+function vocabulary(
+  definitions: readonly Readonly<{
+    name: string;
+    description: string;
+    examples: readonly string[];
+  }>[],
+): readonly ManifestDefinition[] {
+  return Object.freeze(
+    definitions.map((definition) =>
+      Object.freeze({
+        name: definition.name,
+        description: definition.description,
+        examples: Object.freeze([...definition.examples]),
+      }),
+    ),
+  );
+}

package/src/catalog/errors.ts

@@ -1,6 +1,31 @@
+import type { JsonObject } from "./json";
+
+export type CatalogErrorCode =
+  | "lookup"
+  | "invalid_input"
+  | "integrity"
+  | "unavailable_capability"
+  | "incompatible_profile"
+  | "embedding_failure"
+  | "operation_failed"
+  | "response_too_large";
+
+export type CatalogErrorOptions = Readonly<{
+  code?: CatalogErrorCode;
+  details?: JsonObject;
+  hints?: readonly string[];
+}>;
+
 export class CatalogError extends Error {
-  constructor(message: string) {
+  readonly code: CatalogErrorCode;
+  readonly details: Readonly<JsonObject>;
+  readonly hints: readonly string[];
+
+  constructor(message: string, options: CatalogErrorOptions = {}) {
     super(message);
     this.name = "CatalogError";
+    this.code = options.code ?? "invalid_input";
+    this.details = Object.freeze({ ...options.details });
+    this.hints = Object.freeze([...(options.hints ?? [])]);
   }
 }