@yejiming/dsh-data-agent 0.0.13 → 0.1.1

package/lib/types/catalog-adapters.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Dialect-aware technical metadata adapters. Every statement is package-owned,
+ * read-only, bounded by the shared connection runner, and returns normalized
+ * observations rather than CLI text.
+ */
+import type { DataAgentConnections, DatabaseConnection, DatabaseType } from './connections.ts';
+import type { CatalogObservation, CatalogRelation, CatalogScope } from './catalog-types.ts';
+export interface CatalogAdapterOptions {
+    maxTextChars: number;
+    schemaConcurrency: number;
+    assetConcurrency: number;
+}
+export interface CatalogAdapterContext {
+    connections: DataAgentConnections;
+    connection: DatabaseConnection;
+    sessionId: string;
+    sourceId: string;
+    runId: string;
+    scope: CatalogScope;
+    signal: AbortSignal;
+    options: CatalogAdapterOptions;
+    onProgress?(kind: 'schema' | 'relation' | 'field'): void;
+}
+export interface CatalogAdapterResult {
+    observations: CatalogObservation[];
+    relations: CatalogRelation[];
+    coverageComplete: boolean;
+    unavailableScopes: string[];
+}
+export interface CatalogAdapter {
+    readonly type: DatabaseType;
+    readonly capabilities: Readonly<Record<string, 'supported' | 'unsupported' | 'unavailable'>>;
+    scan(context: CatalogAdapterContext): Promise<CatalogAdapterResult>;
+}
+interface RawMetadataRow {
+    rowKind: 'relation' | 'column' | 'primary_key' | 'foreign_key' | 'index';
+    schema: string;
+    relation: string;
+    relationType: string;
+    relationComment: string;
+    column: string;
+    dataType: string;
+    nullable: string;
+    columnComment: string;
+    ordinal: string;
+}
+/** Registry contains an explicit adapter entry for every supported dialect. */
+export declare function createCatalogAdapterRegistry(): Readonly<Record<DatabaseType, CatalogAdapter>>;
+/** Pure SQL constructor used by fixture tests; values are SQL literals, never identifiers. */
+export declare function buildCatalogMetadataSql(type: Exclude<DatabaseType, 'hive' | 'impala'>, database: string, schema: string, table?: string): string;
+export declare function parseCatalogMetadataRows(type: DatabaseType, stdout: string): RawMetadataRow[];
+export {};

package/lib/types/catalog-ai.d.ts

@@ -0,0 +1,49 @@
+/** DSH-native AI enrichment for table and field business-meaning candidates. */
+import type { AgentRegistry } from '@deepseek-ai/dsh-agent';
+import { type LlmCallConfig, type LlmRuntime } from '@deepseek-ai/dsh-llm';
+export interface CatalogModelSelection {
+    provider: string;
+    model: string;
+    reasoningEffort?: LlmCallConfig['reasoningEffort'];
+}
+export interface CatalogMeaningFieldInput {
+    assetId: string;
+    name: string;
+    dataType?: string;
+    nullable?: boolean;
+    comment?: string;
+    keyKinds: string[];
+}
+export interface CatalogMeaningTableInput {
+    assetId: string;
+    schema: string;
+    name: string;
+    objectType: 'table' | 'view';
+    comment?: string;
+    fields: CatalogMeaningFieldInput[];
+    relations: Array<{
+        kind: string;
+        name?: string;
+        fromAssetId: string;
+        toAssetId?: string;
+        columnAssetIds: string[];
+        referencedColumnAssetIds?: string[];
+    }>;
+}
+export interface CatalogMeaningModelResult {
+    table: {
+        assetId: string;
+        meaning: string;
+    };
+    fields: Array<{
+        assetId: string;
+        meaning: string;
+    }>;
+}
+export interface CatalogMeaningGenerator {
+    capture(sessionId: string): CatalogModelSelection;
+    generate(selection: CatalogModelSelection, input: CatalogMeaningTableInput, signal: AbortSignal): Promise<CatalogMeaningModelResult>;
+}
+/** Resolve the exact current session model once, then use the host's configured LLM adapters and credentials. */
+export declare function createDshCatalogMeaningGenerator(agents: AgentRegistry, llm: LlmRuntime): CatalogMeaningGenerator;
+export declare function validateModelResult(raw: string, input: CatalogMeaningTableInput): CatalogMeaningModelResult;

package/lib/types/catalog-command.d.ts

@@ -0,0 +1,28 @@
+/** Preset-scoped `/catalog` human command adapter. */
+import type { Context } from '@deepseek-ai/cordis';
+import type { CommandInvocation, CommandResult } from '@deepseek-ai/dsh-commands';
+import type { CatalogScope } from './catalog-types.ts';
+export declare const CATALOG_COMMAND_USAGE: string;
+export type CatalogCommandAction = {
+    kind: 'scan';
+    scope?: CatalogScope;
+} | {
+    kind: 'status';
+    runId?: string;
+} | {
+    kind: 'cancel';
+    runId?: string;
+} | {
+    kind: 'diff';
+    fromRunId?: string;
+    toRunId?: string;
+} | {
+    kind: 'view';
+};
+export interface CatalogCommandPresentation {
+    watch(run: Awaited<ReturnType<Context['dataAgentCatalogScanner']['start']>>): void;
+    open(sessionId: string): boolean;
+}
+export declare function registerCatalogCommand(ctx: Context, presentation?: CatalogCommandPresentation): () => void;
+export declare function executeCatalogCommand(ctx: Context, invocation: CommandInvocation, presentation?: CatalogCommandPresentation): Promise<CommandResult>;
+export declare function parseCatalogAction(rawInput: string): CatalogCommandAction;

package/lib/types/catalog-identity.d.ts

@@ -0,0 +1,23 @@
+/** Deterministic identity, normalization, and fingerprint helpers. */
+import type { DatabaseType } from './database-types.ts';
+import type { CatalogIdentity, CatalogTechnicalPayload } from './catalog-types.ts';
+/** Strip unsafe controls, normalize Unicode, and enforce one explicit bound. */
+export declare function normalizeCatalogText(value: string, maxChars: number): {
+    value: string;
+    truncated: boolean;
+};
+/** Normalize an observed identifier according to the dialect's identity rules. */
+export declare function normalizeCatalogIdentifier(type: DatabaseType, value: string): string;
+/** v1 source identity is deliberately the stable connection profile id. */
+export declare function catalogSourceId(profileId: string): string;
+/** Build canonical structured identity without parsing display paths. */
+export declare function canonicalCatalogIdentity(type: DatabaseType, identity: CatalogIdentity): CatalogIdentity;
+/** Stable opaque asset id derived only from structured identity components. */
+export declare function catalogAssetId(type: DatabaseType, identity: CatalogIdentity): string;
+export declare function catalogSemanticId(sourceId: string, kind: 'term' | 'metric', name: string): string;
+/** Technical fingerprint excludes run-specific provenance and display-only truncation facts. */
+export declare function catalogTechnicalFingerprint(payload: CatalogTechnicalPayload, status?: string): string;
+/** Deterministic JSON encoding for hashes and cursor order keys. */
+export declare function stableJson(value: unknown): string;
+export declare function catalogRevisionId(assetId: string, revision: number): string;
+export declare function catalogSemanticRevisionId(semanticId: string, version: number): string;

package/lib/types/catalog-storage.d.ts

@@ -0,0 +1,265 @@
+/** Durable versioned Catalog storage-domain and persistence adapter. */
+import { type Domain } from '@deepseek-ai/dsh-storage-domain';
+import { type CatalogAssetHead, type CatalogAssetRevision, type CatalogIndexRecord, type CatalogIndexState, type CatalogObservation, type CatalogRelation, type CatalogRun, type CatalogSemanticEntry, type CatalogSemanticRevision, type CatalogSource } from './catalog-types.ts';
+export declare const CATALOG_STORAGE_DOMAIN = "data_agent_catalog";
+export declare const CATALOG_STORAGE_VERSION = 1;
+/** Strict schemas reject secret-shaped or raw-result fields at the durable boundary. */
+export declare const catalogStorageSpec: {
+    name: string;
+    version: number;
+    tables: {
+        sources: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            id: string;
+            profileId: string;
+            type: "mysql" | "postgres" | "sqlite" | "oracle" | "hive" | "impala" | "clickhouse" | "doris" | "sqlserver";
+            name: string;
+            database: string;
+            credentialConfigured: boolean;
+            createdAt: string;
+            updatedAt: string;
+            host?: string | undefined;
+            lastFullScanAt?: string | undefined;
+            lastPartialScanAt?: string | undefined;
+        }>;
+        scan_runs: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            id: string;
+            sourceId: string;
+            sessionId: string;
+            scope: {
+                kind: "source";
+            } | {
+                kind: "schema";
+                schema: string;
+            } | {
+                kind: "table";
+                schema: string;
+                table: string;
+            };
+            status: "queued" | "running" | "applying" | "succeeded" | "failed" | "cancelled" | "interrupted";
+            coverageComplete: boolean;
+            progress: {
+                schemas: number;
+                relations: number;
+                fields: number;
+                assets: number;
+            };
+            createdAt: string;
+            startedAt?: string | undefined;
+            completedAt?: string | undefined;
+            error?: string | undefined;
+            enrichment?: {
+                status: "queued" | "running" | "succeeded" | "failed" | "cancelled" | "partial";
+                provider: string;
+                model: string;
+                tablesTotal: number;
+                tablesCompleted: number;
+                tablesFailed: number;
+                candidatesGenerated: number;
+                reasoningEffort?: string | undefined;
+                startedAt?: string | undefined;
+                completedAt?: string | undefined;
+                error?: string | undefined;
+            } | undefined;
+        }>;
+        observations: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            runId: string;
+            sourceId: string;
+            assetId: string;
+            status: "observed" | "missing" | "unavailable";
+            fingerprint: string;
+            observedAt: string;
+            payload: {
+                identity: {
+                    sourceId: string;
+                    database: string;
+                    schema: string;
+                    kind: "table" | "schema" | "view" | "column" | "primary_key" | "foreign_key" | "index";
+                    name: string;
+                    relation?: string | undefined;
+                };
+                name: string;
+                path: string;
+                provenance: {
+                    source: "database";
+                    dialect: "mysql" | "postgres" | "sqlite" | "oracle" | "hive" | "impala" | "clickhouse" | "doris" | "sqlserver";
+                    runId: string;
+                };
+                parentId?: string | undefined;
+                objectType?: "table" | "view" | undefined;
+                dataType?: string | undefined;
+                nullable?: boolean | undefined;
+                ordinal?: number | undefined;
+                comment?: string | undefined;
+                referencedAssetIds?: string[] | undefined;
+                attributes?: Record<string, string | number | boolean | null> | undefined;
+                capabilities?: Record<string, "unavailable" | "supported" | "unsupported"> | undefined;
+                truncatedFields?: string[] | undefined;
+            };
+        }>;
+        asset_revisions: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            id: string;
+            assetId: string;
+            sourceId: string;
+            runId: string;
+            revision: number;
+            status: "observed" | "missing" | "unavailable";
+            fingerprint: string;
+            observedAt: string;
+            changeSummary: string[];
+            payload: {
+                identity: {
+                    sourceId: string;
+                    database: string;
+                    schema: string;
+                    kind: "table" | "schema" | "view" | "column" | "primary_key" | "foreign_key" | "index";
+                    name: string;
+                    relation?: string | undefined;
+                };
+                name: string;
+                path: string;
+                provenance: {
+                    source: "database";
+                    dialect: "mysql" | "postgres" | "sqlite" | "oracle" | "hive" | "impala" | "clickhouse" | "doris" | "sqlserver";
+                    runId: string;
+                };
+                parentId?: string | undefined;
+                objectType?: "table" | "view" | undefined;
+                dataType?: string | undefined;
+                nullable?: boolean | undefined;
+                ordinal?: number | undefined;
+                comment?: string | undefined;
+                referencedAssetIds?: string[] | undefined;
+                attributes?: Record<string, string | number | boolean | null> | undefined;
+                capabilities?: Record<string, "unavailable" | "supported" | "unsupported"> | undefined;
+                truncatedFields?: string[] | undefined;
+            };
+            previousRevisionId?: string | undefined;
+        }>;
+        asset_heads: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            assetId: string;
+            sourceId: string;
+            revisionIds: string[];
+            firstSeenAt: string;
+            lastSeenAt: string;
+        }>;
+        relations: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            id: string;
+            sourceId: string;
+            runId: string;
+            kind: "primary_key" | "foreign_key" | "index" | "parent";
+            fromAssetId: string;
+            columnAssetIds: string[];
+            observedAt: string;
+            toAssetId?: string | undefined;
+            name?: string | undefined;
+            referencedColumnAssetIds?: string[] | undefined;
+        }>;
+        semantic_entries: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            id: string;
+            sourceId: string;
+            kind: "metric" | "meaning" | "term";
+            currentVersion: number;
+            createdAt: string;
+            updatedAt: string;
+        }>;
+        semantic_revisions: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, {
+            id: string;
+            semanticId: string;
+            sourceId: string;
+            version: number;
+            createdAt: string;
+            definition: {
+                name: string;
+                aliases: string[];
+                description: string;
+                sourceAssetIds: string[];
+                status: "inferred" | "verified" | "needs_review" | "retired";
+                kind: "term";
+                owner?: string | undefined;
+                validFrom?: string | undefined;
+                validTo?: string | undefined;
+                revisionNote?: string | undefined;
+                verifiedAt?: string | undefined;
+                needsReviewReason?: string | undefined;
+                triggerRunId?: string | undefined;
+            } | {
+                formula: string;
+                grain: string;
+                filters: string[];
+                exclusions: string[];
+                name: string;
+                aliases: string[];
+                description: string;
+                sourceAssetIds: string[];
+                status: "inferred" | "verified" | "needs_review" | "retired";
+                kind: "metric";
+                timeFieldAssetId?: string | undefined;
+                owner?: string | undefined;
+                validFrom?: string | undefined;
+                validTo?: string | undefined;
+                revisionNote?: string | undefined;
+                verifiedAt?: string | undefined;
+                needsReviewReason?: string | undefined;
+                triggerRunId?: string | undefined;
+            } | {
+                targetAssetId: string;
+                targetKind: "table" | "view" | "column";
+                generatedBy: {
+                    kind: "ai";
+                    provider: string;
+                    model: string;
+                    runId: string;
+                };
+                name: string;
+                aliases: string[];
+                description: string;
+                sourceAssetIds: string[];
+                status: "inferred" | "verified" | "needs_review" | "retired";
+                kind: "meaning";
+                owner?: string | undefined;
+                validFrom?: string | undefined;
+                validTo?: string | undefined;
+                revisionNote?: string | undefined;
+                verifiedAt?: string | undefined;
+                needsReviewReason?: string | undefined;
+                triggerRunId?: string | undefined;
+            };
+        }>;
+        search_index: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, CatalogIndexRecord>;
+        index_state: import("@deepseek-ai/dsh-storage-domain").DomainTableSpec<string, CatalogIndexState>;
+    };
+};
+export type CatalogStorageDomain = Domain<typeof catalogStorageSpec>;
+export interface CatalogPersistence {
+    getSource(id: string): CatalogSource | undefined;
+    listSources(): CatalogSource[];
+    putSource(source: CatalogSource): Promise<void>;
+    getRun(id: string): CatalogRun | undefined;
+    listRuns(sourceId?: string): CatalogRun[];
+    putRun(run: CatalogRun): Promise<void>;
+    putObservation(observation: CatalogObservation): Promise<void>;
+    listObservations(runId: string): CatalogObservation[];
+    deleteObservations(runId: string): Promise<void>;
+    getAssetHead(assetId: string): CatalogAssetHead | undefined;
+    listAssetHeads(sourceId?: string): CatalogAssetHead[];
+    putAssetHead(head: CatalogAssetHead): Promise<void>;
+    getAssetRevision(id: string): CatalogAssetRevision | undefined;
+    listAssetRevisions(assetId?: string): CatalogAssetRevision[];
+    putAssetRevision(revision: CatalogAssetRevision): Promise<void>;
+    listRelations(sourceId?: string): CatalogRelation[];
+    putRelation(relation: CatalogRelation): Promise<void>;
+    getSemanticEntry(id: string): CatalogSemanticEntry | undefined;
+    listSemanticEntries(sourceId?: string): CatalogSemanticEntry[];
+    putSemanticEntry(entry: CatalogSemanticEntry): Promise<void>;
+    getSemanticRevision(id: string): CatalogSemanticRevision | undefined;
+    listSemanticRevisions(semanticId?: string): CatalogSemanticRevision[];
+    putSemanticRevision(revision: CatalogSemanticRevision): Promise<void>;
+    listIndex(sourceId?: string): CatalogIndexRecord[];
+    putIndex(record: CatalogIndexRecord): Promise<void>;
+    clearIndex(sourceId?: string): Promise<void>;
+    getIndexState(): CatalogIndexState | undefined;
+    putIndexState(state: CatalogIndexState): Promise<void>;
+}
+export declare function createDomainCatalogPersistence(domain: CatalogStorageDomain): CatalogPersistence;
+/** In-memory adapter used when Catalog persistence is explicitly disabled and by focused tests. */
+export declare function createMemoryCatalogPersistence(): CatalogPersistence;

package/lib/types/catalog-tools.d.ts

@@ -0,0 +1,5 @@
+/** Read-only Catalog model tools. No scan or review service is injected here. */
+import type { Context } from '@deepseek-ai/cordis';
+import { type JsonValue } from '@deepseek-ai/dsh-tools';
+export declare function applyCatalogTools(ctx: Context): void;
+export declare function sanitizeToolValue(value: unknown): JsonValue;

package/lib/types/catalog-tui.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Optional dsh-tui presentation adapter for Catalog progress and read-only
+ * browsing. It only consumes the public `tuiStatus`/`tuiScenes` service
+ * shapes and deliberately has no runtime import from dsh-tui or React.
+ */
+import type { Context } from '@deepseek-ai/cordis';
+import type { CatalogCommandPresentation } from './catalog-command.ts';
+import type { CatalogAssetDetail, CatalogRun } from './catalog-types.ts';
+export interface CatalogTuiAdapter extends CatalogCommandPresentation {
+    dispose(): void;
+}
+/** Create an adapter only from public optional services exposed by dsh-tui. */
+export declare function createCatalogTuiAdapter(ctx: Context): CatalogTuiAdapter;
+/** One bounded status-line projection shared by the adapter and tests. */
+export declare function formatCatalogTuiStatus(run: CatalogRun): string;
+export declare function isCatalogRunSettled(run: CatalogRun): boolean;
+/** Text projection for the independently scrollable right pane. */
+export declare function buildCatalogTuiDetailLines(detail: CatalogAssetDetail): string[];