@kiyeonjeon21/datacontext 0.3.2 → 0.4.0

package/src/core/harvester.ts

@@ -1,22 +1,56 @@
 /**
  * Auto-Harvester
- * Automatically collects metadata from database (COMMENT, indexes, etc.)
+ * 
+ * Automatically collects metadata from database and syncs to Knowledge Store.
  * Platform-agnostic: works with MCP, REST API, SDK, etc.
+ * 
+ * ## What Gets Harvested
+ * 
+ * - **Table/Column Comments**: COMMENT ON TABLE/COLUMN statements
+ * - **Indexes**: Index definitions and types
+ * - **Foreign Keys**: FK constraints → TableRelationship edges
+ * - **Row Counts**: Estimated table sizes
+ * 
+ * ## Relationship Harvesting (NEW in v2.0)
+ * 
+ * Foreign keys are now automatically converted to TableRelationship
+ * entries in the Knowledge Graph:
+ * 
+ * ```
+ * FK: orders.user_id → users.id
+ *       ↓
+ * TableRelationship {
+ *   from: { table: 'orders', schema: 'public' },
+ *   to: { table: 'users', schema: 'public' },
+ *   joinCondition: 'orders.user_id = users.id',
+ *   relationshipType: 'foreign_key',
+ *   cardinality: 'many-to-one'
+ * }
+ * ```
+ * 
+ * @module core/harvester
  */
 
 import type { DatabaseAdapter } from '../adapters/base.js';
 import type { KnowledgeStore } from '../knowledge/store.js';
 import type { HarvestedMetadata } from './types.js';
+import type { TableRelationship } from '../knowledge/types.js';
+import { createKnowledgeMeta, createTableRef } from '../knowledge/types.js';
 
+/**
+ * Harvester configuration options.
+ */
 export interface HarvesterConfig {
   /** Include table/column comments */
   includeComments: boolean;
   /** Include index information */
   includeIndexes: boolean;
-  /** Include foreign key information */
+  /** Include foreign key information and convert to relationships */
   includeForeignKeys: boolean;
   /** Include estimated row counts */
   includeRowCounts: boolean;
+  /** Automatically create relationships from FK (default: true) */
+  createRelationshipsFromFK: boolean;
 }
 
 const DEFAULT_CONFIG: HarvesterConfig = {
@@ -24,6 +58,7 @@ const DEFAULT_CONFIG: HarvesterConfig = {
   includeIndexes: true,
   includeForeignKeys: true,
   includeRowCounts: true,
+  createRelationshipsFromFK: true,
 };
 
 /**
@@ -117,18 +152,32 @@ export class Harvester {
   }
 
   /**
-   * Sync harvested metadata to knowledge store
-   * Only adds new entries, doesn't overwrite user-defined ones
+   * Sync harvested metadata to knowledge store.
+   * 
+   * Performs the following operations:
+   * 1. Adds table descriptions from DB COMMENT (if not already defined)
+   * 2. Adds column descriptions from DB COMMENT (if not already defined)
+   * 3. Converts FK constraints to TableRelationship entries (NEW in v2.0)
+   * 
+   * Only adds new entries - doesn't overwrite user-defined descriptions.
+   * 
+   * @param schema - Schema to harvest (defaults to 'public')
+   * @returns Summary of what was added
    */
   async syncToKnowledge(schema: string = 'public'): Promise<{
     tablesProcessed: number;
     descriptionsAdded: number;
     columnsAdded: number;
+    relationshipsAdded: number;
   }> {
     const harvested = await this.harvestSchema(schema);
     
     let descriptionsAdded = 0;
     let columnsAdded = 0;
+    let relationshipsAdded = 0;
+
+    // Collect all relationships first for batch insertion
+    const relationshipsToAdd: TableRelationship[] = [];
 
     for (const metadata of harvested) {
       // Check if table already has a user-defined description
@@ -139,9 +188,6 @@ export class Harvester {
         await this.knowledge.setTableDescription(metadata.tableName, metadata.tableComment, {
           schema: metadata.schema,
         });
-        
-        // Mark as auto-harvested by updating the source
-        // The knowledge store will have source: 'user' by default, we need to track this
         descriptionsAdded++;
       }
 
@@ -160,6 +206,26 @@ export class Harvester {
           columnsAdded++;
         }
       }
+
+      // Convert FK to relationships (NEW in v2.0)
+      if (this.config.createRelationshipsFromFK && metadata.foreignKeys) {
+        for (const fk of metadata.foreignKeys) {
+          const relationship = this.createRelationshipFromFK(
+            metadata.tableName,
+            metadata.schema,
+            fk.column,
+            fk.referencedTable,
+            fk.referencedColumn,
+            fk.constraintName
+          );
+          relationshipsToAdd.push(relationship);
+        }
+      }
+    }
+
+    // Batch add relationships
+    if (relationshipsToAdd.length > 0) {
+      relationshipsAdded = await this.knowledge.addRelationships(relationshipsToAdd);
     }
 
     // Save changes
@@ -169,20 +235,64 @@ export class Harvester {
       tablesProcessed: harvested.length,
       descriptionsAdded,
       columnsAdded,
+      relationshipsAdded,
+    };
+  }
+
+  /**
+   * Create a TableRelationship from FK information.
+   * 
+   * @internal
+   */
+  private createRelationshipFromFK(
+    fromTable: string,
+    fromSchema: string,
+    fromColumn: string,
+    toTable: string,
+    toColumn: string,
+    constraintName?: string
+  ): TableRelationship {
+    const schemaHash = this.knowledge.getSchemaHash();
+    
+    return {
+      ...createKnowledgeMeta('auto', schemaHash),
+      type: 'table_relationship',
+      from: createTableRef(fromTable, fromSchema),
+      to: createTableRef(toTable, fromSchema), // Assume same schema for FK target
+      relationshipType: 'foreign_key',
+      joinCondition: `${fromTable}.${fromColumn} = ${toTable}.${toColumn}`,
+      cardinality: 'many-to-one', // Default for FK
+      fromColumns: [fromColumn],
+      toColumns: [toColumn],
+      isPreferred: true,
+      constraintName,
     };
   }
 
   /**
-   * Get harvesting summary without syncing
+   * Get harvesting summary without syncing.
+   * 
+   * Provides a preview of what would be harvested.
    */
   async getSummary(schema: string = 'public'): Promise<{
     tables: number;
     tablesWithComments: number;
     columnsWithComments: number;
     tablesWithForeignKeys: number;
+    totalForeignKeys: number;
+    existingRelationships: number;
   }> {
     const harvested = await this.harvestSchema(schema);
     
+    // Count total FKs
+    const totalFKs = harvested.reduce(
+      (sum, h) => sum + (h.foreignKeys?.length ?? 0),
+      0
+    );
+
+    // Count existing relationships
+    const existingRels = this.knowledge.getRelationships().length;
+
     return {
       tables: harvested.length,
       tablesWithComments: harvested.filter(h => h.tableComment).length,
@@ -191,6 +301,8 @@ export class Harvester {
         0
       ),
       tablesWithForeignKeys: harvested.filter(h => h.foreignKeys && h.foreignKeys.length > 0).length,
+      totalForeignKeys: totalFKs,
+      existingRelationships: existingRels,
     };
   }
 }

package/src/core/types.ts

@@ -78,26 +78,40 @@ export interface QueryFeedback {
 }
 
 /**
- * Harvested metadata from database
+ * Harvested metadata from database.
+ * 
+ * This is the raw data collected from database introspection.
+ * Foreign keys are subsequently converted to TableRelationship entries
+ * by the Harvester during sync.
+ * 
+ * @see TableRelationship for the graph edge representation
  */
 export interface HarvestedMetadata {
   /** Table name */
   tableName: string;
   /** Schema */
   schema: string;
-  /** Table comment from DB */
+  /** Table comment from DB (COMMENT ON TABLE) */
   tableComment?: string;
-  /** Column comments from DB */
+  /** Column comments from DB (COMMENT ON COLUMN) */
   columnComments: Record<string, string>;
   /** Estimated row count */
   estimatedRowCount?: number;
-  /** Indexes */
+  /** Indexes (formatted as strings for display) */
   indexes?: string[];
-  /** Foreign keys */
+  /** 
+   * Foreign keys (raw data).
+   * These are converted to TableRelationship entries during harvesting.
+   */
   foreignKeys?: Array<{
+    /** Source column name */
     column: string;
+    /** Referenced table name */
     referencedTable: string;
+    /** Referenced column name */
     referencedColumn: string;
+    /** FK constraint name (optional) */
+    constraintName?: string;
   }>;
 }
 
@@ -175,6 +189,8 @@ export interface DataContextConfig {
     includeIndexes: boolean;
     includeForeignKeys: boolean;
     includeRowCounts: boolean;
+    /** Convert FK constraints to TableRelationship entries (default: true) */
+    createRelationshipsFromFK?: boolean;
   };
   /** Metrics configuration */
   metrics?: {

package/src/index.ts

@@ -58,8 +58,26 @@ export type { TableInfo, ColumnInfo, SchemaInfo } from './schema/types.js';
 // === Knowledge Types ===
 export type { 
   TableDescription, 
+  TableRelationship,
+  TableRef,
+  RelationshipType,
+  RelationshipCardinality,
   QueryExample, 
   BusinessRule,
-  KnowledgeData 
+  BusinessTerm,
+  KnowledgeData,
+  KnowledgeEntry,
+  KnowledgeMeta,
+  KnowledgeSource,
+} from './knowledge/types.js';
+
+// === Knowledge Factory Functions ===
+export {
+  createTableRef,
+  tableRefToString,
+  tableRefsEqual,
+  createRelationshipFromFK,
+  createKnowledgeMeta,
+  generateId,
 } from './knowledge/types.js';