@kiyeonjeon21/datacontext 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/sqlite.d.ts.map +1 -1
- package/dist/adapters/sqlite.js +13 -0
- package/dist/adapters/sqlite.js.map +1 -1
- package/dist/api/server.d.ts.map +1 -1
- package/dist/api/server.js +115 -0
- package/dist/api/server.js.map +1 -1
- package/dist/cli/index.js +58 -14
- package/dist/cli/index.js.map +1 -1
- package/dist/core/context-service.d.ts +63 -0
- package/dist/core/context-service.d.ts.map +1 -1
- package/dist/core/context-service.js +66 -0
- package/dist/core/context-service.js.map +1 -1
- package/dist/core/harvester.d.ts +57 -5
- package/dist/core/harvester.d.ts.map +1 -1
- package/dist/core/harvester.js +86 -6
- package/dist/core/harvester.js.map +1 -1
- package/dist/core/types.d.ts +21 -5
- package/dist/core/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge/store.d.ts +186 -3
- package/dist/knowledge/store.d.ts.map +1 -1
- package/dist/knowledge/store.js +389 -5
- package/dist/knowledge/store.js.map +1 -1
- package/dist/knowledge/types.d.ts +252 -4
- package/dist/knowledge/types.d.ts.map +1 -1
- package/dist/knowledge/types.js +138 -1
- package/dist/knowledge/types.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +231 -3
- package/dist/mcp/tools.js.map +1 -1
- package/docs/KNOWLEDGE_GRAPH.md +540 -0
- package/docs/KNOWLEDGE_TYPES.md +261 -0
- package/docs/MULTI_DB_ARCHITECTURE.md +319 -0
- package/package.json +1 -1
- package/scripts/create-sqlite-testdb.sh +75 -0
- package/scripts/test-databases.sh +324 -0
- package/sqlite:./test-sqlite.db +0 -0
- package/src/adapters/sqlite.ts +16 -0
- package/src/api/server.ts +134 -0
- package/src/cli/index.ts +57 -16
- package/src/core/context-service.ts +70 -0
- package/src/core/harvester.ts +120 -8
- package/src/core/types.ts +21 -5
- package/src/index.ts +19 -1
- package/src/knowledge/store.ts +480 -6
- package/src/knowledge/types.ts +321 -4
- package/src/mcp/tools.ts +273 -3
- package/test-sqlite.db +0 -0
- package/tests/knowledge-store.test.ts +130 -0
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# Test Databases Setup Script
|
|
3
|
+
# Uses Podman to run PostgreSQL, MySQL, and MariaDB for testing
|
|
4
|
+
|
|
5
|
+
set -e
|
|
6
|
+
|
|
7
|
+
# Colors
|
|
8
|
+
RED='\033[0;31m'
|
|
9
|
+
GREEN='\033[0;32m'
|
|
10
|
+
YELLOW='\033[1;33m'
|
|
11
|
+
NC='\033[0m' # No Color
|
|
12
|
+
|
|
13
|
+
echo -e "${GREEN}=== DataContext Database Test Environment ===${NC}"
|
|
14
|
+
|
|
15
|
+
# Check if podman is available
|
|
16
|
+
if ! command -v podman &> /dev/null; then
|
|
17
|
+
echo -e "${RED}Error: podman is not installed${NC}"
|
|
18
|
+
echo "Install with: brew install podman"
|
|
19
|
+
exit 1
|
|
20
|
+
fi
|
|
21
|
+
|
|
22
|
+
# Configuration
|
|
23
|
+
POSTGRES_PORT=5433
|
|
24
|
+
MYSQL_PORT=3307
|
|
25
|
+
MARIADB_PORT=3308
|
|
26
|
+
NETWORK_NAME="datacontext-test"
|
|
27
|
+
|
|
28
|
+
# Create network if not exists
|
|
29
|
+
podman network exists $NETWORK_NAME 2>/dev/null || podman network create $NETWORK_NAME
|
|
30
|
+
|
|
31
|
+
# Function to start PostgreSQL
|
|
32
|
+
start_postgres() {
|
|
33
|
+
echo -e "\n${YELLOW}Starting PostgreSQL...${NC}"
|
|
34
|
+
|
|
35
|
+
# Stop if running
|
|
36
|
+
podman rm -f datacontext-postgres 2>/dev/null || true
|
|
37
|
+
|
|
38
|
+
podman run -d \
|
|
39
|
+
--name datacontext-postgres \
|
|
40
|
+
--network $NETWORK_NAME \
|
|
41
|
+
-e POSTGRES_USER=postgres \
|
|
42
|
+
-e POSTGRES_PASSWORD=postgres \
|
|
43
|
+
-e POSTGRES_DB=datacontext_test \
|
|
44
|
+
-p $POSTGRES_PORT:5432 \
|
|
45
|
+
postgres:15-alpine
|
|
46
|
+
|
|
47
|
+
echo -e "${GREEN}PostgreSQL started on port $POSTGRES_PORT${NC}"
|
|
48
|
+
echo "Connection: postgres://postgres:postgres@localhost:$POSTGRES_PORT/datacontext_test"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
# Function to start MySQL
|
|
52
|
+
start_mysql() {
|
|
53
|
+
echo -e "\n${YELLOW}Starting MySQL...${NC}"
|
|
54
|
+
|
|
55
|
+
# Stop if running
|
|
56
|
+
podman rm -f datacontext-mysql 2>/dev/null || true
|
|
57
|
+
|
|
58
|
+
podman run -d \
|
|
59
|
+
--name datacontext-mysql \
|
|
60
|
+
--network $NETWORK_NAME \
|
|
61
|
+
-e MYSQL_ROOT_PASSWORD=mysql \
|
|
62
|
+
-e MYSQL_DATABASE=datacontext_test \
|
|
63
|
+
-e MYSQL_USER=mysql \
|
|
64
|
+
-e MYSQL_PASSWORD=mysql \
|
|
65
|
+
-p $MYSQL_PORT:3306 \
|
|
66
|
+
mysql:8.0
|
|
67
|
+
|
|
68
|
+
echo -e "${GREEN}MySQL started on port $MYSQL_PORT${NC}"
|
|
69
|
+
echo "Connection: mysql://mysql:mysql@localhost:$MYSQL_PORT/datacontext_test"
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
# Function to start MariaDB
|
|
73
|
+
start_mariadb() {
|
|
74
|
+
echo -e "\n${YELLOW}Starting MariaDB...${NC}"
|
|
75
|
+
|
|
76
|
+
# Stop if running
|
|
77
|
+
podman rm -f datacontext-mariadb 2>/dev/null || true
|
|
78
|
+
|
|
79
|
+
podman run -d \
|
|
80
|
+
--name datacontext-mariadb \
|
|
81
|
+
--network $NETWORK_NAME \
|
|
82
|
+
-e MARIADB_ROOT_PASSWORD=mariadb \
|
|
83
|
+
-e MARIADB_DATABASE=datacontext_test \
|
|
84
|
+
-e MARIADB_USER=mariadb \
|
|
85
|
+
-e MARIADB_PASSWORD=mariadb \
|
|
86
|
+
-p $MARIADB_PORT:3306 \
|
|
87
|
+
mariadb:10.11
|
|
88
|
+
|
|
89
|
+
echo -e "${GREEN}MariaDB started on port $MARIADB_PORT${NC}"
|
|
90
|
+
echo "Connection: mysql://mariadb:mariadb@localhost:$MARIADB_PORT/datacontext_test"
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
# Function to wait for DB
|
|
94
|
+
wait_for_db() {
|
|
95
|
+
local name=$1
|
|
96
|
+
local port=$2
|
|
97
|
+
local max_attempts=30
|
|
98
|
+
local attempt=0
|
|
99
|
+
|
|
100
|
+
echo -e "${YELLOW}Waiting for $name to be ready...${NC}"
|
|
101
|
+
|
|
102
|
+
while [ $attempt -lt $max_attempts ]; do
|
|
103
|
+
if podman exec $name echo "SELECT 1" 2>/dev/null; then
|
|
104
|
+
echo -e "${GREEN}$name is ready!${NC}"
|
|
105
|
+
return 0
|
|
106
|
+
fi
|
|
107
|
+
attempt=$((attempt + 1))
|
|
108
|
+
sleep 1
|
|
109
|
+
done
|
|
110
|
+
|
|
111
|
+
echo -e "${RED}$name failed to start${NC}"
|
|
112
|
+
return 1
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
# Function to create test data
|
|
116
|
+
create_test_data() {
|
|
117
|
+
local db_type=$1
|
|
118
|
+
local container=$2
|
|
119
|
+
|
|
120
|
+
echo -e "\n${YELLOW}Creating test data for $db_type...${NC}"
|
|
121
|
+
|
|
122
|
+
case $db_type in
|
|
123
|
+
postgres)
|
|
124
|
+
podman exec -i $container psql -U postgres -d datacontext_test << 'EOF'
|
|
125
|
+
-- Users table
|
|
126
|
+
CREATE TABLE IF NOT EXISTS users (
|
|
127
|
+
id SERIAL PRIMARY KEY,
|
|
128
|
+
name VARCHAR(100) NOT NULL,
|
|
129
|
+
email VARCHAR(255) UNIQUE NOT NULL,
|
|
130
|
+
status INTEGER DEFAULT 1,
|
|
131
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
132
|
+
deleted_at TIMESTAMP
|
|
133
|
+
);
|
|
134
|
+
|
|
135
|
+
COMMENT ON TABLE users IS 'User accounts table';
|
|
136
|
+
COMMENT ON COLUMN users.status IS 'Account status: 0=inactive, 1=active, 2=suspended';
|
|
137
|
+
|
|
138
|
+
-- Products table
|
|
139
|
+
CREATE TABLE IF NOT EXISTS products (
|
|
140
|
+
id SERIAL PRIMARY KEY,
|
|
141
|
+
name VARCHAR(200) NOT NULL,
|
|
142
|
+
price DECIMAL(10,2) NOT NULL,
|
|
143
|
+
inventory INTEGER DEFAULT 0,
|
|
144
|
+
category VARCHAR(50),
|
|
145
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
COMMENT ON TABLE products IS 'Product catalog';
|
|
149
|
+
|
|
150
|
+
-- Orders table
|
|
151
|
+
CREATE TABLE IF NOT EXISTS orders (
|
|
152
|
+
id SERIAL PRIMARY KEY,
|
|
153
|
+
user_id INTEGER REFERENCES users(id),
|
|
154
|
+
product_id INTEGER REFERENCES products(id),
|
|
155
|
+
quantity INTEGER NOT NULL,
|
|
156
|
+
total_amount DECIMAL(10,2) NOT NULL,
|
|
157
|
+
status VARCHAR(20) DEFAULT 'pending',
|
|
158
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
159
|
+
);
|
|
160
|
+
|
|
161
|
+
COMMENT ON TABLE orders IS 'Customer orders';
|
|
162
|
+
COMMENT ON COLUMN orders.status IS 'Order status: pending, paid, shipped, delivered, cancelled';
|
|
163
|
+
|
|
164
|
+
-- Insert sample data
|
|
165
|
+
INSERT INTO users (name, email, status) VALUES
|
|
166
|
+
('Alice', 'alice@example.com', 1),
|
|
167
|
+
('Bob', 'bob@example.com', 1),
|
|
168
|
+
('Charlie', 'charlie@example.com', 0),
|
|
169
|
+
('Diana', 'diana@example.com', 2)
|
|
170
|
+
ON CONFLICT DO NOTHING;
|
|
171
|
+
|
|
172
|
+
INSERT INTO products (name, price, inventory, category) VALUES
|
|
173
|
+
('Laptop', 999.99, 50, 'electronics'),
|
|
174
|
+
('Mouse', 29.99, 200, 'electronics'),
|
|
175
|
+
('Keyboard', 79.99, 100, 'electronics'),
|
|
176
|
+
('Desk Chair', 299.99, 30, 'furniture')
|
|
177
|
+
ON CONFLICT DO NOTHING;
|
|
178
|
+
|
|
179
|
+
INSERT INTO orders (user_id, product_id, quantity, total_amount, status) VALUES
|
|
180
|
+
(1, 1, 1, 999.99, 'delivered'),
|
|
181
|
+
(1, 2, 2, 59.98, 'delivered'),
|
|
182
|
+
(2, 3, 1, 79.99, 'shipped'),
|
|
183
|
+
(2, 4, 1, 299.99, 'pending')
|
|
184
|
+
ON CONFLICT DO NOTHING;
|
|
185
|
+
|
|
186
|
+
SELECT 'PostgreSQL test data created!' as result;
|
|
187
|
+
EOF
|
|
188
|
+
;;
|
|
189
|
+
|
|
190
|
+
mysql)
|
|
191
|
+
sleep 10 # MySQL takes longer to start
|
|
192
|
+
podman exec -i $container mysql -umysql -pmysql datacontext_test << 'EOF'
|
|
193
|
+
-- Users table
|
|
194
|
+
CREATE TABLE IF NOT EXISTS users (
|
|
195
|
+
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
196
|
+
name VARCHAR(100) NOT NULL,
|
|
197
|
+
email VARCHAR(255) UNIQUE NOT NULL,
|
|
198
|
+
status INT DEFAULT 1,
|
|
199
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
200
|
+
deleted_at TIMESTAMP NULL
|
|
201
|
+
) COMMENT='User accounts table';
|
|
202
|
+
|
|
203
|
+
-- Products table
|
|
204
|
+
CREATE TABLE IF NOT EXISTS products (
|
|
205
|
+
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
206
|
+
name VARCHAR(200) NOT NULL,
|
|
207
|
+
price DECIMAL(10,2) NOT NULL,
|
|
208
|
+
inventory INT DEFAULT 0,
|
|
209
|
+
category VARCHAR(50),
|
|
210
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
211
|
+
) COMMENT='Product catalog';
|
|
212
|
+
|
|
213
|
+
-- Orders table
|
|
214
|
+
CREATE TABLE IF NOT EXISTS orders (
|
|
215
|
+
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
216
|
+
user_id INT,
|
|
217
|
+
product_id INT,
|
|
218
|
+
quantity INT NOT NULL,
|
|
219
|
+
total_amount DECIMAL(10,2) NOT NULL,
|
|
220
|
+
status VARCHAR(20) DEFAULT 'pending',
|
|
221
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
222
|
+
FOREIGN KEY (user_id) REFERENCES users(id),
|
|
223
|
+
FOREIGN KEY (product_id) REFERENCES products(id)
|
|
224
|
+
) COMMENT='Customer orders';
|
|
225
|
+
|
|
226
|
+
-- Insert sample data
|
|
227
|
+
INSERT IGNORE INTO users (name, email, status) VALUES
|
|
228
|
+
('Alice', 'alice@example.com', 1),
|
|
229
|
+
('Bob', 'bob@example.com', 1),
|
|
230
|
+
('Charlie', 'charlie@example.com', 0),
|
|
231
|
+
('Diana', 'diana@example.com', 2);
|
|
232
|
+
|
|
233
|
+
INSERT IGNORE INTO products (name, price, inventory, category) VALUES
|
|
234
|
+
('Laptop', 999.99, 50, 'electronics'),
|
|
235
|
+
('Mouse', 29.99, 200, 'electronics'),
|
|
236
|
+
('Keyboard', 79.99, 100, 'electronics'),
|
|
237
|
+
('Desk Chair', 299.99, 30, 'furniture');
|
|
238
|
+
|
|
239
|
+
INSERT IGNORE INTO orders (user_id, product_id, quantity, total_amount, status) VALUES
|
|
240
|
+
(1, 1, 1, 999.99, 'delivered'),
|
|
241
|
+
(1, 2, 2, 59.98, 'delivered'),
|
|
242
|
+
(2, 3, 1, 79.99, 'shipped'),
|
|
243
|
+
(2, 4, 1, 299.99, 'pending');
|
|
244
|
+
|
|
245
|
+
SELECT 'MySQL test data created!' as result;
|
|
246
|
+
EOF
|
|
247
|
+
;;
|
|
248
|
+
esac
|
|
249
|
+
|
|
250
|
+
echo -e "${GREEN}Test data created for $db_type${NC}"
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
# Function to stop all
|
|
254
|
+
stop_all() {
|
|
255
|
+
echo -e "\n${YELLOW}Stopping all test databases...${NC}"
|
|
256
|
+
podman rm -f datacontext-postgres datacontext-mysql datacontext-mariadb 2>/dev/null || true
|
|
257
|
+
echo -e "${GREEN}All databases stopped${NC}"
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
# Function to show status
|
|
261
|
+
show_status() {
|
|
262
|
+
echo -e "\n${GREEN}=== Database Status ===${NC}"
|
|
263
|
+
podman ps --filter "name=datacontext-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
# Function to show connection strings
|
|
267
|
+
show_connections() {
|
|
268
|
+
echo -e "\n${GREEN}=== Connection Strings ===${NC}"
|
|
269
|
+
echo ""
|
|
270
|
+
echo -e "${YELLOW}PostgreSQL:${NC}"
|
|
271
|
+
echo " postgres://postgres:postgres@localhost:$POSTGRES_PORT/datacontext_test"
|
|
272
|
+
echo ""
|
|
273
|
+
echo -e "${YELLOW}MySQL:${NC}"
|
|
274
|
+
echo " mysql://mysql:mysql@localhost:$MYSQL_PORT/datacontext_test"
|
|
275
|
+
echo ""
|
|
276
|
+
echo -e "${YELLOW}MariaDB:${NC}"
|
|
277
|
+
echo " mysql://mariadb:mariadb@localhost:$MARIADB_PORT/datacontext_test"
|
|
278
|
+
echo ""
|
|
279
|
+
echo -e "${GREEN}=== DataContext Commands ===${NC}"
|
|
280
|
+
echo ""
|
|
281
|
+
echo "# Start servers"
|
|
282
|
+
echo "npx @kiyeonjeon21/datacontext serve postgres://postgres:postgres@localhost:$POSTGRES_PORT/datacontext_test --port 3000"
|
|
283
|
+
echo "npx @kiyeonjeon21/datacontext serve mysql://mysql:mysql@localhost:$MYSQL_PORT/datacontext_test --port 3001"
|
|
284
|
+
echo ""
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
# Parse arguments
|
|
288
|
+
case "${1:-start}" in
|
|
289
|
+
start)
|
|
290
|
+
start_postgres
|
|
291
|
+
start_mysql
|
|
292
|
+
sleep 15 # Wait for DBs to be ready
|
|
293
|
+
create_test_data postgres datacontext-postgres
|
|
294
|
+
create_test_data mysql datacontext-mysql
|
|
295
|
+
show_status
|
|
296
|
+
show_connections
|
|
297
|
+
;;
|
|
298
|
+
postgres)
|
|
299
|
+
start_postgres
|
|
300
|
+
sleep 5
|
|
301
|
+
create_test_data postgres datacontext-postgres
|
|
302
|
+
show_connections
|
|
303
|
+
;;
|
|
304
|
+
mysql)
|
|
305
|
+
start_mysql
|
|
306
|
+
sleep 15
|
|
307
|
+
create_test_data mysql datacontext-mysql
|
|
308
|
+
show_connections
|
|
309
|
+
;;
|
|
310
|
+
stop)
|
|
311
|
+
stop_all
|
|
312
|
+
;;
|
|
313
|
+
status)
|
|
314
|
+
show_status
|
|
315
|
+
show_connections
|
|
316
|
+
;;
|
|
317
|
+
*)
|
|
318
|
+
echo "Usage: $0 {start|postgres|mysql|stop|status}"
|
|
319
|
+
exit 1
|
|
320
|
+
;;
|
|
321
|
+
esac
|
|
322
|
+
|
|
323
|
+
echo -e "\n${GREEN}Done!${NC}"
|
|
324
|
+
|
|
File without changes
|
package/src/adapters/sqlite.ts
CHANGED
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import Database from 'better-sqlite3';
|
|
14
|
+
import { existsSync, mkdirSync } from 'fs';
|
|
15
|
+
import { dirname, resolve } from 'path';
|
|
14
16
|
import type { DatabaseAdapter, QueryResult, ConnectionConfig } from './base.js';
|
|
15
17
|
import type { SchemaInfo, TableInfo, ColumnInfo, IndexInfo, ForeignKeyInfo } from '../schema/types.js';
|
|
16
18
|
import { hashSchema } from '../knowledge/schema-hash.js';
|
|
@@ -41,6 +43,20 @@ export class SQLiteAdapter implements DatabaseAdapter {
|
|
|
41
43
|
}
|
|
42
44
|
|
|
43
45
|
try {
|
|
46
|
+
// Resolve relative paths to absolute
|
|
47
|
+
let resolvedPath = this.filePath;
|
|
48
|
+
if (this.filePath !== ':memory:') {
|
|
49
|
+
resolvedPath = resolve(process.cwd(), this.filePath);
|
|
50
|
+
|
|
51
|
+
// Ensure parent directory exists
|
|
52
|
+
const dir = dirname(resolvedPath);
|
|
53
|
+
if (!existsSync(dir)) {
|
|
54
|
+
mkdirSync(dir, { recursive: true });
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
this.filePath = resolvedPath;
|
|
58
|
+
}
|
|
59
|
+
|
|
44
60
|
this.db = new Database(this.filePath, { readonly: false });
|
|
45
61
|
// Enable foreign keys
|
|
46
62
|
this.db.pragma('foreign_keys = ON');
|
package/src/api/server.ts
CHANGED
|
@@ -6,6 +6,39 @@
|
|
|
6
6
|
import express, { Request, Response, NextFunction } from 'express';
|
|
7
7
|
import cors from 'cors';
|
|
8
8
|
import type { DataContextService } from '../core/context-service.js';
|
|
9
|
+
import type { TableRelationship } from '../knowledge/types.js';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Format join path response for API
|
|
13
|
+
*/
|
|
14
|
+
function formatPathResponse(
|
|
15
|
+
fromTable: string,
|
|
16
|
+
toTable: string,
|
|
17
|
+
path: TableRelationship[]
|
|
18
|
+
): unknown {
|
|
19
|
+
const steps = path.map((rel, idx) => ({
|
|
20
|
+
step: idx + 1,
|
|
21
|
+
from: `${rel.from.schema}.${rel.from.table}`,
|
|
22
|
+
to: `${rel.to.schema}.${rel.to.table}`,
|
|
23
|
+
joinCondition: rel.joinCondition,
|
|
24
|
+
cardinality: rel.cardinality || 'unknown',
|
|
25
|
+
}));
|
|
26
|
+
|
|
27
|
+
// Generate SQL hint
|
|
28
|
+
const sqlParts = path.map(rel =>
|
|
29
|
+
`JOIN ${rel.to.table} ON ${rel.joinCondition}`
|
|
30
|
+
);
|
|
31
|
+
const sqlHint = `FROM ${fromTable}\n${sqlParts.join('\n')}`;
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
found: true,
|
|
35
|
+
from: fromTable,
|
|
36
|
+
to: toTable,
|
|
37
|
+
hops: path.length,
|
|
38
|
+
path: steps,
|
|
39
|
+
sqlHint,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
9
42
|
|
|
10
43
|
export interface ApiServerConfig {
|
|
11
44
|
/** DataContext service instance */
|
|
@@ -403,6 +436,107 @@ export function createApiServer(config: ApiServerConfig): express.Application {
|
|
|
403
436
|
}
|
|
404
437
|
});
|
|
405
438
|
|
|
439
|
+
// ============================================================
|
|
440
|
+
// Knowledge Graph Endpoints
|
|
441
|
+
// ============================================================
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Get all relationships from Knowledge Graph
|
|
445
|
+
* GET /api/relationships?table=orders
|
|
446
|
+
*/
|
|
447
|
+
app.get('/api/relationships', (_req: Request, res: Response) => {
|
|
448
|
+
try {
|
|
449
|
+
const table = _req.query.table as string | undefined;
|
|
450
|
+
let relationships = service.getRelationships();
|
|
451
|
+
|
|
452
|
+
// Filter by table if specified
|
|
453
|
+
if (table) {
|
|
454
|
+
relationships = relationships.filter(
|
|
455
|
+
rel => rel.from.table === table || rel.to.table === table
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
res.json({
|
|
460
|
+
count: relationships.length,
|
|
461
|
+
relationships: relationships.map(rel => ({
|
|
462
|
+
id: rel.id,
|
|
463
|
+
from: `${rel.from.schema}.${rel.from.table}`,
|
|
464
|
+
to: `${rel.to.schema}.${rel.to.table}`,
|
|
465
|
+
type: rel.relationshipType,
|
|
466
|
+
joinCondition: rel.joinCondition,
|
|
467
|
+
cardinality: rel.cardinality,
|
|
468
|
+
isPreferred: rel.isPreferred,
|
|
469
|
+
})),
|
|
470
|
+
});
|
|
471
|
+
} catch (error) {
|
|
472
|
+
res.status(500).json({
|
|
473
|
+
error: error instanceof Error ? error.message : 'Failed to get relationships'
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* Find join path between two tables
|
|
480
|
+
* GET /api/relationships/path?from=order_items&to=users&schema=public
|
|
481
|
+
*/
|
|
482
|
+
app.get('/api/relationships/path', (_req: Request, res: Response) => {
|
|
483
|
+
try {
|
|
484
|
+
const fromTable = _req.query.from as string;
|
|
485
|
+
const toTable = _req.query.to as string;
|
|
486
|
+
const schema = (_req.query.schema as string) || 'public';
|
|
487
|
+
|
|
488
|
+
if (!fromTable || !toTable) {
|
|
489
|
+
res.status(400).json({ error: 'Both "from" and "to" query parameters are required' });
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
const path = service.findJoinPath(fromTable, toTable, schema);
|
|
494
|
+
|
|
495
|
+
if (path.length === 0) {
|
|
496
|
+
// Try 'main' schema for SQLite
|
|
497
|
+
const pathMain = service.findJoinPath(fromTable, toTable, 'main');
|
|
498
|
+
if (pathMain.length === 0) {
|
|
499
|
+
res.json({
|
|
500
|
+
found: false,
|
|
501
|
+
from: fromTable,
|
|
502
|
+
to: toTable,
|
|
503
|
+
path: [],
|
|
504
|
+
message: `No path found between '${fromTable}' and '${toTable}'.`,
|
|
505
|
+
});
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
res.json(formatPathResponse(fromTable, toTable, pathMain));
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
res.json(formatPathResponse(fromTable, toTable, path));
|
|
513
|
+
} catch (error) {
|
|
514
|
+
res.status(500).json({
|
|
515
|
+
error: error instanceof Error ? error.message : 'Failed to find join path'
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Get Knowledge Graph summary
|
|
522
|
+
* GET /api/graph/summary
|
|
523
|
+
*/
|
|
524
|
+
app.get('/api/graph/summary', (_req: Request, res: Response) => {
|
|
525
|
+
try {
|
|
526
|
+
const summary = service.getGraphSummary();
|
|
527
|
+
res.json({
|
|
528
|
+
...summary,
|
|
529
|
+
message: summary.edgeCount > 0
|
|
530
|
+
? `Knowledge Graph has ${summary.edgeCount} relationships connecting ${summary.tablesWithRelationships} tables.`
|
|
531
|
+
: 'Knowledge Graph is empty. Run POST /api/harvest to discover relationships.',
|
|
532
|
+
});
|
|
533
|
+
} catch (error) {
|
|
534
|
+
res.status(500).json({
|
|
535
|
+
error: error instanceof Error ? error.message : 'Failed to get graph summary'
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
|
|
406
540
|
// ============================================================
|
|
407
541
|
// Glossary (Business Terms) Endpoints
|
|
408
542
|
// ============================================================
|
package/src/cli/index.ts
CHANGED
|
@@ -34,9 +34,14 @@ function createAdapter(connectionString: string): DatabaseAdapter {
|
|
|
34
34
|
return createPostgresAdapter(connectionString);
|
|
35
35
|
} else if (url.startsWith('mysql://') || url.startsWith('mariadb://')) {
|
|
36
36
|
return createMySQLAdapter(connectionString);
|
|
37
|
-
} else if (url.startsWith('sqlite://') || url.endsWith('.sqlite') || url.endsWith('.sqlite3') || url.endsWith('.db') || url === ':memory:') {
|
|
38
|
-
// SQLite: sqlite://path
|
|
39
|
-
|
|
37
|
+
} else if (url.startsWith('sqlite://') || url.startsWith('sqlite:') || url.endsWith('.sqlite') || url.endsWith('.sqlite3') || url.endsWith('.db') || url === ':memory:') {
|
|
38
|
+
// SQLite: sqlite://path or sqlite:path or ./mydb.sqlite or :memory:
|
|
39
|
+
let filePath = connectionString;
|
|
40
|
+
if (url.startsWith('sqlite://')) {
|
|
41
|
+
filePath = connectionString.replace(/^sqlite:\/\//i, '');
|
|
42
|
+
} else if (url.startsWith('sqlite:')) {
|
|
43
|
+
filePath = connectionString.replace(/^sqlite:/i, '');
|
|
44
|
+
}
|
|
40
45
|
return createSQLiteAdapter(filePath);
|
|
41
46
|
} else {
|
|
42
47
|
throw new Error(
|
|
@@ -65,20 +70,37 @@ program
|
|
|
65
70
|
.option('--allowed-tables <tables...>', 'Only allow access to these tables')
|
|
66
71
|
.action(async (connectionString: string, options) => {
|
|
67
72
|
try {
|
|
68
|
-
// Parse connection string
|
|
69
|
-
const config = parseConnectionString(connectionString);
|
|
70
|
-
|
|
71
73
|
// Create adapter (auto-detect database type)
|
|
72
74
|
const adapter = createAdapter(connectionString);
|
|
73
75
|
await adapter.connect();
|
|
74
76
|
|
|
77
|
+
// Determine database ID for knowledge store
|
|
78
|
+
let databaseId: string;
|
|
79
|
+
const url = connectionString.toLowerCase();
|
|
80
|
+
|
|
81
|
+
if (url.startsWith('sqlite://') || url.startsWith('sqlite:') || url.endsWith('.sqlite') || url.endsWith('.sqlite3') || url.endsWith('.db') || url === ':memory:') {
|
|
82
|
+
// SQLite: use filename as database ID
|
|
83
|
+
let filePath = connectionString;
|
|
84
|
+
if (url.startsWith('sqlite://')) {
|
|
85
|
+
filePath = connectionString.replace(/^sqlite:\/\//i, '');
|
|
86
|
+
} else if (url.startsWith('sqlite:')) {
|
|
87
|
+
filePath = connectionString.replace(/^sqlite:/i, '');
|
|
88
|
+
}
|
|
89
|
+
const fileName = filePath.split('/').pop() || 'sqlite';
|
|
90
|
+
databaseId = `sqlite_${fileName.replace(/[^a-zA-Z0-9]/g, '_')}`;
|
|
91
|
+
} else {
|
|
92
|
+
// URL-based databases (postgres, mysql)
|
|
93
|
+
const config = parseConnectionString(connectionString);
|
|
94
|
+
databaseId = `${config.host}_${config.database}`;
|
|
95
|
+
}
|
|
96
|
+
|
|
75
97
|
// Create knowledge store
|
|
76
|
-
const databaseId = `${config.host}_${config.database}`;
|
|
77
98
|
const knowledge = createKnowledgeStore(databaseId);
|
|
78
99
|
await knowledge.load();
|
|
79
100
|
|
|
80
|
-
// Update schema hash
|
|
81
|
-
const
|
|
101
|
+
// Update schema hash (SQLite uses 'main' schema)
|
|
102
|
+
const schemaName = url.startsWith('sqlite') || url.endsWith('.db') ? 'main' : options.schema;
|
|
103
|
+
const schema = await adapter.getSchema(schemaName);
|
|
82
104
|
knowledge.updateSchemaHash(schema.schemaHash);
|
|
83
105
|
await knowledge.save();
|
|
84
106
|
|
|
@@ -836,21 +858,40 @@ program
|
|
|
836
858
|
try {
|
|
837
859
|
console.log('[datacontext] Starting REST API server...');
|
|
838
860
|
|
|
839
|
-
// Parse connection string
|
|
840
|
-
const config = parseConnectionString(connectionString);
|
|
841
|
-
|
|
842
861
|
// Create adapter (auto-detect database type)
|
|
843
862
|
const adapter = createAdapter(connectionString);
|
|
844
863
|
await adapter.connect();
|
|
845
|
-
|
|
864
|
+
|
|
865
|
+
// Determine database ID for knowledge store
|
|
866
|
+
let databaseId: string;
|
|
867
|
+
const url = connectionString.toLowerCase();
|
|
868
|
+
|
|
869
|
+
if (url.startsWith('sqlite://') || url.startsWith('sqlite:') || url.endsWith('.sqlite') || url.endsWith('.sqlite3') || url.endsWith('.db') || url === ':memory:') {
|
|
870
|
+
// SQLite: use filename as database ID
|
|
871
|
+
let filePath = connectionString;
|
|
872
|
+
if (url.startsWith('sqlite://')) {
|
|
873
|
+
filePath = connectionString.replace(/^sqlite:\/\//i, '');
|
|
874
|
+
} else if (url.startsWith('sqlite:')) {
|
|
875
|
+
filePath = connectionString.replace(/^sqlite:/i, '');
|
|
876
|
+
}
|
|
877
|
+
const fileName = filePath.split('/').pop() || 'sqlite';
|
|
878
|
+
databaseId = `sqlite_${fileName.replace(/[^a-zA-Z0-9]/g, '_')}`;
|
|
879
|
+
console.log(`[datacontext] Connected to SQLite: ${filePath}`);
|
|
880
|
+
} else {
|
|
881
|
+
// URL-based databases (postgres, mysql)
|
|
882
|
+
const config = parseConnectionString(connectionString);
|
|
883
|
+
databaseId = `${config.host}_${config.database}`;
|
|
884
|
+
console.log(`[datacontext] Connected to ${config.database}`);
|
|
885
|
+
}
|
|
846
886
|
|
|
847
887
|
// Create knowledge store
|
|
848
|
-
const databaseId = `${config.host}_${config.database}`;
|
|
849
888
|
const knowledge = createKnowledgeStore(databaseId);
|
|
850
889
|
await knowledge.load();
|
|
851
890
|
|
|
852
|
-
// Update schema hash
|
|
853
|
-
const
|
|
891
|
+
// Update schema hash (SQLite uses 'main' schema)
|
|
892
|
+
const isSqlite = url.startsWith('sqlite') || url.endsWith('.db') || url.endsWith('.sqlite') || url.endsWith('.sqlite3');
|
|
893
|
+
const schemaName = isSqlite ? 'main' : options.schema;
|
|
894
|
+
const schema = await adapter.getSchema(schemaName);
|
|
854
895
|
knowledge.updateSchemaHash(schema.schemaHash);
|
|
855
896
|
await knowledge.save();
|
|
856
897
|
|
|
@@ -577,6 +577,76 @@ export class DataContextService {
|
|
|
577
577
|
return this.harvester.getSummary(schema);
|
|
578
578
|
}
|
|
579
579
|
|
|
580
|
+
// ============================================================
|
|
581
|
+
// Knowledge Graph
|
|
582
|
+
// ============================================================
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Get all relationships from the Knowledge Graph.
|
|
586
|
+
*
|
|
587
|
+
* Returns table relationships (foreign keys, joins) that have been
|
|
588
|
+
* discovered through harvesting or manually defined.
|
|
589
|
+
*
|
|
590
|
+
* @returns Array of table relationships
|
|
591
|
+
*
|
|
592
|
+
* @example
|
|
593
|
+
* ```typescript
|
|
594
|
+
* const relationships = service.getRelationships();
|
|
595
|
+
* for (const rel of relationships) {
|
|
596
|
+
* console.log(`${rel.from.table} → ${rel.to.table}: ${rel.joinCondition}`);
|
|
597
|
+
* }
|
|
598
|
+
* ```
|
|
599
|
+
*/
|
|
600
|
+
getRelationships() {
|
|
601
|
+
return this.knowledge.getRelationships();
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Find the optimal join path between two tables.
|
|
606
|
+
*
|
|
607
|
+
* Uses BFS traversal of the Knowledge Graph to find the shortest
|
|
608
|
+
* path between tables. Useful for generating JOIN conditions.
|
|
609
|
+
*
|
|
610
|
+
* @param fromTable - Source table name
|
|
611
|
+
* @param toTable - Destination table name
|
|
612
|
+
* @param schema - Schema name (defaults to 'public')
|
|
613
|
+
* @returns Array of relationships forming the path, empty if no path exists
|
|
614
|
+
*
|
|
615
|
+
* @example
|
|
616
|
+
* ```typescript
|
|
617
|
+
* // Find path from order_items to users
|
|
618
|
+
* const path = service.findJoinPath('order_items', 'users', 'public');
|
|
619
|
+
* // Returns: [
|
|
620
|
+
* // { from: order_items, to: orders, join: order_items.order_id = orders.id },
|
|
621
|
+
* // { from: orders, to: users, join: orders.user_id = users.id }
|
|
622
|
+
* // ]
|
|
623
|
+
*
|
|
624
|
+
* // Generate SQL
|
|
625
|
+
* const joins = path.map(r => `JOIN ${r.to.table} ON ${r.joinCondition}`).join('\n');
|
|
626
|
+
* ```
|
|
627
|
+
*/
|
|
628
|
+
findJoinPath(fromTable: string, toTable: string, schema: string = 'public') {
|
|
629
|
+
return this.knowledge.findJoinPath(fromTable, toTable, schema);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Get Knowledge Graph summary statistics.
|
|
634
|
+
*
|
|
635
|
+
* Returns information about the graph structure including
|
|
636
|
+
* node counts, edge counts, and connectivity.
|
|
637
|
+
*
|
|
638
|
+
* @returns Graph summary object
|
|
639
|
+
*
|
|
640
|
+
* @example
|
|
641
|
+
* ```typescript
|
|
642
|
+
* const summary = service.getGraphSummary();
|
|
643
|
+
* console.log(`Nodes: ${summary.nodeCount}, Edges: ${summary.edgeCount}`);
|
|
644
|
+
* ```
|
|
645
|
+
*/
|
|
646
|
+
getGraphSummary() {
|
|
647
|
+
return this.knowledge.getGraphSummary();
|
|
648
|
+
}
|
|
649
|
+
|
|
580
650
|
// ============================================================
|
|
581
651
|
// Feedback & Learning
|
|
582
652
|
// ============================================================
|