@digital-science-dsl/dimensions-analytics-mcp 0.5.3

package/src/dsl/response-parser.ts

@@ -0,0 +1,299 @@
+/**
+ * Response parser for Dimensions API facet results.
+ * Transforms raw API responses into typed facet result structures.
+ * @module response-parser
+ */
+
+import { z } from "zod";
+import { ValidationError } from "../client/index.js";
+import type { BaseBucket } from "./types/buckets.js";
+
+/**
+ * Schema for validating bucket objects in facet results.
+ * Buckets must have an id (string or number) and a non-negative count.
+ */
+const BaseBucketSchema = z
+  .object({
+    id: z.union([z.string(), z.number()]),
+    count: z.number().int().nonnegative(),
+  })
+  .passthrough();
+
+/**
+ * Schema for validating _stats object in API responses.
+ */
+const StatsSchema = z
+  .object({
+    total_count: z.number().int().nonnegative(),
+  })
+  .passthrough();
+
+/**
+ * Configuration for a facet field, including optional aggregation indicators.
+ */
+export interface FacetFieldConfig {
+  /** Optional array of aggregation indicator names to validate */
+  readonly indicators?: readonly string[];
+}
+
+/**
+ * Parses entity data from a Dimensions API response.
+ * Validates that the entity array is properly structured.
+ * Falls back to data.length if _stats.total_count is not available.
+ *
+ * @template T - The entity type
+ * @param response - Raw API response object
+ * @param entityKey - The entity type key (e.g., "publications")
+ * @returns Entity result with data array and totalCount
+ * @throws {ValidationError} If entity data is not an array
+ *
+ * @example
+ * ```typescript
+ * const result = parseEntityResponse<Publication>(response, "publications");
+ * console.log(result.data.length, result.totalCount);
+ * ```
+ */
+export function parseEntityResponse<T>(
+  response: Record<string, unknown>,
+  entityKey: string,
+): EntityResult<T> {
+  const entityData = response[entityKey];
+
+  // If entity key is missing, return empty result
+  if (entityData === undefined) {
+    return { data: [], totalCount: 0 };
+  }
+
+  // Validate entity data is an array
+  if (!Array.isArray(entityData)) {
+    throw new ValidationError(`Entity key "${entityKey}" must be an array`, {
+      entityKey,
+      actualType: typeof entityData,
+    });
+  }
+
+  const entityArray = entityData as T[];
+
+  // Try to get total_count from stats, fallback to data length
+  const statsResult = StatsSchema.safeParse(response._stats);
+  const totalCount = statsResult.success ? statsResult.data.total_count : entityArray.length;
+
+  return {
+    data: entityArray,
+    totalCount,
+  };
+}
+
+/**
+ * Cache for bucket schemas keyed by sorted indicator combination.
+ * Avoids rebuilding the same ZodObject on every transformResponse call.
+ */
+const bucketSchemaCache = new Map<string, z.ZodObject<z.ZodRawShape>>();
+
+/**
+ * Creates a bucket schema that validates both base fields and aggregation indicators.
+ * Caches schemas by indicator combination to avoid repeated allocations.
+ * @param indicators - Array of indicator names that must be present as numbers
+ * @returns Zod schema for bucket validation
+ */
+function createBucketSchema(indicators?: readonly string[]) {
+  if (!indicators || indicators.length === 0) {
+    return BaseBucketSchema;
+  }
+
+  const cacheKey = [...indicators].sort().join(",");
+  const cached = bucketSchemaCache.get(cacheKey);
+  if (cached) {
+    return cached;
+  }
+
+  // Build a schema that requires each indicator to be a number
+  const indicatorSchema: Record<string, z.ZodNumber> = {};
+  for (const indicator of indicators) {
+    indicatorSchema[indicator] = z.number();
+  }
+
+  const schema = BaseBucketSchema.extend(indicatorSchema);
+  bucketSchemaCache.set(cacheKey, schema);
+  return schema;
+}
+
+/**
+ * Typed facet result with bucket array.
+ * @template TBucket - The bucket type for this facet
+ */
+export interface TypedFacetResult<TBucket extends BaseBucket = BaseBucket> {
+  readonly field: string;
+  readonly buckets: readonly TBucket[];
+}
+
+/**
+ * Entity data result with array and total count.
+ * @template T - The entity type
+ */
+export interface EntityResult<T> {
+  readonly data: readonly T[];
+  readonly totalCount: number;
+}
+
+/**
+ * Combined result for queries with facets and/or entities.
+ * @template T - The entity type
+ * @template TFacets - Record of facet field names to their typed results
+ */
+export interface FacetQueryResult<
+  T,
+  TFacets extends Record<string, TypedFacetResult> = Record<string, TypedFacetResult>,
+> {
+  readonly entities?: EntityResult<T>;
+  readonly facets: TFacets;
+  /** Raw API response. Optional to avoid doubling memory for large responses. */
+  readonly _raw?: Record<string, unknown>;
+}
+
+/**
+ * Options for parsing facet responses.
+ */
+export interface ParseFacetOptions {
+  /**
+   * Whether to include the raw API response in the result.
+   * Set to false to avoid doubling memory usage for large responses.
+   * @default true
+   */
+  readonly includeRaw?: boolean;
+}
+
+/**
+ * Parses a Dimensions API response into typed facet results.
+ * Extracts entity data and facet buckets from the response.
+ *
+ * @template T - The entity type
+ * @template TFacets - Record of facet field names to their typed results
+ * @param response - Raw API response object
+ * @param entityKey - The entity type key (e.g., "publications")
+ * @param facetFields - Array of facet field names or record of field configs
+ * @param options - Optional parsing configuration
+ * @returns Typed result with entities and facets
+ *
+ * @example
+ * ```typescript
+ * // Simple facets (field names only)
+ * const result = parseFacetResponse<Publication>(
+ *   response,
+ *   "publications",
+ *   ["year", "type"]
+ * );
+ *
+ * // Aggregated facets (with indicator validation)
+ * const result = parseFacetResponse<Publication>(
+ *   response,
+ *   "publications",
+ *   {
+ *     year: {},
+ *     funders: { indicators: ["rcr_avg", "citations_avg"] }
+ *   }
+ * );
+ *
+ * // Exclude raw response to save memory
+ * const result = parseFacetResponse<Publication>(
+ *   response,
+ *   "publications",
+ *   ["year"],
+ *   { includeRaw: false }
+ * );
+ * ```
+ */
+export function parseFacetResponse<
+  T,
+  TFacets extends Record<string, TypedFacetResult> = Record<string, TypedFacetResult>,
+>(
+  response: Record<string, unknown>,
+  entityKey: string,
+  facetFields: readonly string[] | Record<string, FacetFieldConfig>,
+  options: ParseFacetOptions = {},
+): FacetQueryResult<T, TFacets> {
+  const { includeRaw = true } = options;
+  const facets: Record<string, TypedFacetResult> = {};
+
+  // Normalize facetFields to a record
+  const facetConfigs: Record<string, FacetFieldConfig> = Array.isArray(facetFields)
+    ? Object.fromEntries(facetFields.map((f) => [f, {}]))
+    : facetFields;
+
+  // Parse and validate entity data if present
+  let entities: EntityResult<T> | undefined;
+  if (response[entityKey] !== undefined) {
+    if (!Array.isArray(response[entityKey])) {
+      throw new ValidationError(`Entity key "${entityKey}" must be an array`, {
+        entityKey,
+        actualType: typeof response[entityKey],
+      });
+    }
+
+    const entityArray = response[entityKey] as T[];
+    if (entityArray.length > 0) {
+      const statsResult = StatsSchema.safeParse(response._stats);
+      if (!statsResult.success) {
+        throw new ValidationError("API response missing _stats.total_count for entity results", {
+          entityKey,
+          statsErrors: statsResult.error.issues,
+        });
+      }
+      entities = {
+        data: entityArray,
+        totalCount: statsResult.data.total_count,
+      };
+    } else {
+      // Empty array - use 0 as totalCount
+      entities = {
+        data: entityArray,
+        totalCount: 0,
+      };
+    }
+  }
+
+  // Parse and validate facet data
+  for (const [field, config] of Object.entries(facetConfigs)) {
+    const fieldData = response[field];
+
+    if (fieldData === undefined) {
+      throw new ValidationError(`Missing facet field "${field}" in API response`, {
+        requestedFields: Object.keys(facetConfigs),
+      });
+    }
+
+    if (!Array.isArray(fieldData)) {
+      throw new ValidationError(`Facet field "${field}" must be an array`, {
+        field,
+        actualType: typeof fieldData,
+      });
+    }
+
+    // Validate bucket structure (including indicators if specified)
+    const bucketSchema = createBucketSchema(config.indicators);
+    const parseResult = z.array(bucketSchema).safeParse(fieldData);
+    if (!parseResult.success) {
+      throw new ValidationError(`Invalid bucket data for facet "${field}"`, {
+        field,
+        indicators: config.indicators,
+        errors: parseResult.error.issues,
+      });
+    }
+
+    facets[field] = {
+      field,
+      buckets: parseResult.data as BaseBucket[],
+    };
+  }
+
+  const result: FacetQueryResult<T, TFacets> = {
+    entities,
+    facets: facets as TFacets,
+  };
+
+  if (includeRaw) {
+    return { ...result, _raw: response };
+  }
+
+  return result;
+}

package/src/dsl/schema/cache.ts

@@ -0,0 +1,124 @@
+/**
+ * Disk cache helpers for describe schema responses.
+ * @module schema/cache
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { extractDescribeSchema } from "./extract.js";
+import type { DescribeSchemaResponse } from "./types.js";
+
+/** Default cache TTL: 24 hours. */
+export const DEFAULT_SCHEMA_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+
+/** On-disk cache envelope with metadata. */
+export type SchemaCacheEnvelope = {
+  readonly cachedAt: string;
+  readonly version?: string;
+  readonly schema: DescribeSchemaResponse;
+};
+
+/** Parsed cache entry with normalized timestamps. */
+export type SchemaCacheEntry = {
+  readonly response: DescribeSchemaResponse;
+  readonly cachedAt: Date | undefined;
+  readonly version: string | undefined;
+};
+
+/**
+ * Returns whether a cache entry is within the configured TTL.
+ * @param cachedAt - When the cache was written
+ * @param ttlMs - Maximum cache age in milliseconds
+ * @returns True when the entry is still fresh
+ */
+export function isCacheFresh(cachedAt: Date | undefined, ttlMs: number): boolean {
+  if (cachedAt == null || !Number.isFinite(ttlMs) || ttlMs <= 0) {
+    return false;
+  }
+  return Date.now() - cachedAt.getTime() < ttlMs;
+}
+
+/**
+ * Reads a schema cache file (envelope or legacy bare describe payload).
+ * @param cachePath - Path to JSON cache file
+ * @returns Parsed cache entry or undefined when missing/invalid
+ */
+export async function readSchemaCacheFile(
+  cachePath: string,
+): Promise<SchemaCacheEntry | undefined> {
+  try {
+    const text = await readFile(cachePath, "utf8");
+    const parsed = JSON.parse(text) as unknown;
+    return parseCachePayload(parsed);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Parses cache JSON as an envelope or legacy describe response.
+ * @param parsed - Parsed JSON value
+ * @returns Normalized cache entry
+ */
+export function parseCachePayload(parsed: unknown): SchemaCacheEntry | undefined {
+  if (typeof parsed !== "object" || parsed == null) {
+    return undefined;
+  }
+
+  const record = parsed as Record<string, unknown>;
+
+  if ("schema" in record) {
+    try {
+      const response = extractDescribeSchema(record.schema);
+      const cachedAt = typeof record.cachedAt === "string" ? new Date(record.cachedAt) : undefined;
+      const version = typeof record.version === "string" ? record.version : undefined;
+      return { response, cachedAt, version };
+    } catch {
+      return undefined;
+    }
+  }
+
+  try {
+    const response = extractDescribeSchema(parsed);
+    return { response, cachedAt: undefined, version: undefined };
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Writes describe schema to a cache file with metadata envelope.
+ * @param cachePath - Path to JSON cache file
+ * @param response - Schema response to persist
+ * @param version - Optional DSL version
+ */
+export async function writeSchemaCacheFile(
+  cachePath: string,
+  response: DescribeSchemaResponse,
+  version?: string,
+): Promise<void> {
+  const envelope: SchemaCacheEnvelope = {
+    cachedAt: new Date().toISOString(),
+    ...(version != null ? { version } : {}),
+    schema: response,
+  };
+  await writeFile(cachePath, JSON.stringify(envelope), "utf8");
+}
+
+/**
+ * Resolves cache TTL from options or environment.
+ * @param overrideMs - Explicit TTL from caller
+ * @returns TTL in milliseconds
+ */
+export function resolveCacheTtlMs(overrideMs?: number): number {
+  if (overrideMs != null && Number.isFinite(overrideMs) && overrideMs > 0) {
+    return overrideMs;
+  }
+  const raw = process.env.SCHEMA_CACHE_TTL_MS;
+  if (raw != null && raw.trim() !== "") {
+    const value = Number.parseInt(raw, 10);
+    if (Number.isFinite(value) && value > 0) {
+      return value;
+    }
+  }
+  return DEFAULT_SCHEMA_CACHE_TTL_MS;
+}

package/src/dsl/schema/extract.ts

@@ -0,0 +1,44 @@
+/**
+ * Normalizes raw `describe` API responses.
+ * @module schema/extract
+ */
+
+import type { DescribeSchemaResponse, DescribeVersionResponse } from "./types.js";
+
+/**
+ * Extracts describe schema payload from a raw API response.
+ * @param raw - Raw DSL API response
+ * @returns Normalized describe schema object
+ */
+export function extractDescribeSchema(raw: unknown): DescribeSchemaResponse {
+  if (typeof raw !== "object" || raw === null) {
+    throw new Error("Invalid describe schema response: expected object");
+  }
+  const record = raw as Record<string, unknown>;
+  if ("sources" in record || "entities" in record) {
+    return {
+      sources: (record.sources as DescribeSchemaResponse["sources"]) ?? {},
+      entities: (record.entities as DescribeSchemaResponse["entities"]) ?? {},
+    };
+  }
+  throw new Error("Invalid describe schema response: missing sources/entities");
+}
+
+/**
+ * Extracts version string from `describe version` response.
+ * @param raw - Raw DSL API response
+ * @returns Version string or undefined
+ */
+export function extractDescribeVersion(raw: unknown): string | undefined {
+  if (typeof raw !== "object" || raw === null) {
+    return undefined;
+  }
+  const record = raw as DescribeVersionResponse & Record<string, unknown>;
+  if (typeof record.version === "string") {
+    return record.version;
+  }
+  if (typeof record.release === "string") {
+    return record.release;
+  }
+  return undefined;
+}

package/src/dsl/schema/index.ts

@@ -0,0 +1,50 @@
+/**
+ * Runtime schema from `describe schema` / `describe version`.
+ * @module schema
+ */
+
+export {
+  DEFAULT_SCHEMA_CACHE_TTL_MS,
+  isCacheFresh,
+  parseCachePayload,
+  type SchemaCacheEntry,
+  type SchemaCacheEnvelope,
+} from "./cache.js";
+export { extractDescribeSchema, extractDescribeVersion } from "./extract.js";
+export {
+  clearSchemaCache,
+  getCachedSchemaStore,
+  getOrLoadSchema,
+  type LoadSchemaOptions,
+  loadSchema,
+} from "./load.js";
+export {
+  createSchemaStoreFromResponse,
+  SCHEMA_LIMITS,
+  type SchemaLoadSource,
+  SchemaStore,
+  type SchemaStoreMetadata,
+} from "./store.js";
+export {
+  STRUCTURED_ENTITY_TYPES,
+  type StructuredEntityType,
+  searchResultKey,
+  searchToolName,
+} from "./structured-entities.js";
+export {
+  buildSchemaSummary,
+  SCHEMA_RESOURCE_URIS,
+  type SchemaSourceSummary,
+  type SchemaSummary,
+} from "./summary.js";
+export type {
+  DescribeField,
+  DescribeSchemaResponse,
+  DescribeVersionResponse,
+  EntityDescribe,
+  SourceDescribe,
+} from "./types.js";
+export {
+  assertValidEntity,
+  assertValidSearchIndex,
+} from "./validation.js";

package/src/dsl/schema/load.ts

@@ -0,0 +1,160 @@
+/**
+ * Load schema from the Dimensions API or optional disk cache.
+ * @module schema/load
+ */
+
+import type { DimensionsClient } from "../client.js";
+import {
+  isCacheFresh,
+  readSchemaCacheFile,
+  resolveCacheTtlMs,
+  writeSchemaCacheFile,
+} from "./cache.js";
+import { extractDescribeSchema, extractDescribeVersion } from "./extract.js";
+import { createSchemaStoreFromResponse, type SchemaStore } from "./store.js";
+
+let moduleCache: SchemaStore | undefined;
+let moduleLoadPromise: Promise<SchemaStore> | undefined;
+
+/**
+ * Returns the module-level cached schema store, if loaded.
+ * @returns Cached store or undefined before first load
+ */
+export function getCachedSchemaStore(): SchemaStore | undefined {
+  return moduleCache;
+}
+
+/**
+ * Clears the module-level schema cache (for tests and refresh).
+ */
+export function clearSchemaCache(): void {
+  moduleCache = undefined;
+  moduleLoadPromise = undefined;
+}
+
+export type LoadSchemaOptions = {
+  /** Optional cache file path (defaults to `SCHEMA_CACHE_PATH` env) */
+  cachePath?: string;
+  /** Cache TTL in milliseconds (defaults to `SCHEMA_CACHE_TTL_MS` env or 24h) */
+  cacheTtlMs?: number;
+  /** Skip fresh cache and fetch from the API (stale fallback still applies on failure) */
+  forceRefresh?: boolean;
+  /** Log load stats to stderr (default true) */
+  log?: boolean;
+};
+
+/**
+ * Fetches schema from the Dimensions API, with TTL-aware cache and stale fallback.
+ * @param client - Authenticated Dimensions client
+ * @param options - Cache path, TTL, and logging options
+ * @returns Loaded schema store
+ */
+export async function loadSchema(
+  client: DimensionsClient,
+  options?: LoadSchemaOptions | string,
+): Promise<SchemaStore> {
+  const resolved =
+    typeof options === "string" ? { cachePath: options, log: true as const } : (options ?? {});
+  const cachePath = resolved.cachePath ?? process.env.SCHEMA_CACHE_PATH;
+  const log = resolved.log ?? true;
+  const cacheTtlMs = resolveCacheTtlMs(resolved.cacheTtlMs);
+  const forceRefresh = resolved.forceRefresh ?? false;
+
+  const cached = cachePath ? await readSchemaCacheFile(cachePath) : undefined;
+  const cacheFresh = cached != null && isCacheFresh(cached.cachedAt, cacheTtlMs);
+
+  if (!forceRefresh && cacheFresh && cached) {
+    const store = createSchemaStoreFromResponse(cached.response, cached.version, new Date(), {
+      loadSource: "cache",
+      stale: false,
+      cachedAt: cached.cachedAt,
+    });
+    moduleCache = store;
+    if (log) {
+      logSchemaLoaded(store, "cache (fresh)");
+    }
+    return store;
+  }
+
+  try {
+    const started = performance.now();
+    const [schemaRaw, versionRaw] = await Promise.all([
+      client.rawQuery("describe schema"),
+      client.rawQuery("describe version").catch(() => undefined),
+    ]);
+
+    const response = extractDescribeSchema(schemaRaw);
+    const version = versionRaw ? extractDescribeVersion(versionRaw) : undefined;
+
+    if (cachePath) {
+      await writeSchemaCacheFile(cachePath, response, version);
+    }
+
+    const store = createSchemaStoreFromResponse(response, version, new Date(), {
+      loadSource: "api",
+      stale: false,
+    });
+    moduleCache = store;
+    if (log) {
+      const elapsed = Math.round(performance.now() - started);
+      logSchemaLoaded(store, `api (${elapsed}ms)`);
+    }
+    return store;
+  } catch (error) {
+    if (cached) {
+      const store = createSchemaStoreFromResponse(cached.response, cached.version, new Date(), {
+        loadSource: "cache",
+        stale: true,
+        cachedAt: cached.cachedAt,
+      });
+      moduleCache = store;
+      if (log) {
+        const age =
+          cached.cachedAt != null
+            ? `${Math.round((Date.now() - cached.cachedAt.getTime()) / 60_000)}m old`
+            : "legacy cache";
+        console.error(
+          `[schema] API unavailable; using stale cache (${age}). ` +
+            `Call refresh_schema when the API is reachable.`,
+        );
+        logSchemaLoaded(store, "cache (stale fallback)");
+      }
+      return store;
+    }
+    throw error;
+  }
+}
+
+/**
+ * Returns cached schema or loads once per process.
+ * @param client - Authenticated Dimensions client
+ * @param options - Cache path and logging options
+ * @returns Schema store
+ */
+export async function getOrLoadSchema(
+  client: DimensionsClient,
+  options?: LoadSchemaOptions | string,
+): Promise<SchemaStore> {
+  if (moduleCache) {
+    return moduleCache;
+  }
+  if (!moduleLoadPromise) {
+    moduleLoadPromise = loadSchema(client, options).finally(() => {
+      moduleLoadPromise = undefined;
+    });
+  }
+  return moduleLoadPromise;
+}
+
+function logSchemaLoaded(store: SchemaStore, via: string): void {
+  const stats = store.stats();
+  const flags = [via, store.stale ? "stale" : undefined].filter(Boolean).join(", ");
+  console.error(
+    `[schema] loaded ${stats.sourceCount} sources, ${stats.entityCount} entities` +
+      (stats.dslVersion ? ` (DSL ${stats.dslVersion})` : "") +
+      ` [${flags}]`,
+  );
+}
+
+// Re-export extract helpers for backward compatibility
+export { extractDescribeSchema, extractDescribeVersion } from "./extract.js";