lokicms-plugin-sql 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/federation/cache-layer.d.ts +58 -0
- package/dist/federation/cache-layer.d.ts.map +1 -0
- package/dist/federation/cache-layer.js +210 -0
- package/dist/federation/cache-layer.js.map +1 -0
- package/dist/federation/index.d.ts +12 -0
- package/dist/federation/index.d.ts.map +1 -0
- package/dist/federation/index.js +9 -0
- package/dist/federation/index.js.map +1 -0
- package/dist/federation/source-manager.d.ts +57 -0
- package/dist/federation/source-manager.d.ts.map +1 -0
- package/dist/federation/source-manager.js +238 -0
- package/dist/federation/source-manager.js.map +1 -0
- package/dist/federation/sync-engine.d.ts +68 -0
- package/dist/federation/sync-engine.d.ts.map +1 -0
- package/dist/federation/sync-engine.js +288 -0
- package/dist/federation/sync-engine.js.map +1 -0
- package/dist/index.d.ts +80 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +79 -0
- package/dist/index.js.map +1 -0
- package/dist/plugin.d.ts +28 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +798 -0
- package/dist/plugin.js.map +1 -0
- package/dist/providers/base.d.ts +142 -0
- package/dist/providers/base.d.ts.map +1 -0
- package/dist/providers/base.js +161 -0
- package/dist/providers/base.js.map +1 -0
- package/dist/providers/index.d.ts +22 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +74 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/mariadb.d.ts +83 -0
- package/dist/providers/mariadb.d.ts.map +1 -0
- package/dist/providers/mariadb.js +293 -0
- package/dist/providers/mariadb.js.map +1 -0
- package/dist/providers/mysql.d.ts +78 -0
- package/dist/providers/mysql.d.ts.map +1 -0
- package/dist/providers/mysql.js +284 -0
- package/dist/providers/mysql.js.map +1 -0
- package/dist/providers/postgresql.d.ts +77 -0
- package/dist/providers/postgresql.d.ts.map +1 -0
- package/dist/providers/postgresql.js +296 -0
- package/dist/providers/postgresql.js.map +1 -0
- package/dist/providers/sqlite.d.ts +80 -0
- package/dist/providers/sqlite.d.ts.map +1 -0
- package/dist/providers/sqlite.js +283 -0
- package/dist/providers/sqlite.js.map +1 -0
- package/dist/query/builder.d.ts +74 -0
- package/dist/query/builder.d.ts.map +1 -0
- package/dist/query/builder.js +279 -0
- package/dist/query/builder.js.map +1 -0
- package/dist/query/index.d.ts +10 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +8 -0
- package/dist/query/index.js.map +1 -0
- package/dist/query/transformer.d.ts +74 -0
- package/dist/query/transformer.d.ts.map +1 -0
- package/dist/query/transformer.js +236 -0
- package/dist/query/transformer.js.map +1 -0
- package/dist/types.d.ts +350 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +38 -0
- package/dist/types.js.map +1 -0
- package/dist/vectors/adapter.d.ts +128 -0
- package/dist/vectors/adapter.d.ts.map +1 -0
- package/dist/vectors/adapter.js +79 -0
- package/dist/vectors/adapter.js.map +1 -0
- package/dist/vectors/index.d.ts +41 -0
- package/dist/vectors/index.d.ts.map +1 -0
- package/dist/vectors/index.js +87 -0
- package/dist/vectors/index.js.map +1 -0
- package/dist/vectors/lokijs-vector.d.ts +112 -0
- package/dist/vectors/lokijs-vector.d.ts.map +1 -0
- package/dist/vectors/lokijs-vector.js +217 -0
- package/dist/vectors/lokijs-vector.js.map +1 -0
- package/dist/vectors/mariadb-vector.d.ts +56 -0
- package/dist/vectors/mariadb-vector.d.ts.map +1 -0
- package/dist/vectors/mariadb-vector.js +263 -0
- package/dist/vectors/mariadb-vector.js.map +1 -0
- package/dist/vectors/mysql-vector.d.ts +56 -0
- package/dist/vectors/mysql-vector.d.ts.map +1 -0
- package/dist/vectors/mysql-vector.js +235 -0
- package/dist/vectors/mysql-vector.js.map +1 -0
- package/dist/vectors/pgvector.d.ts +52 -0
- package/dist/vectors/pgvector.d.ts.map +1 -0
- package/dist/vectors/pgvector.js +190 -0
- package/dist/vectors/pgvector.js.map +1 -0
- package/dist/vectors/sqlite-vec.d.ts +80 -0
- package/dist/vectors/sqlite-vec.d.ts.map +1 -0
- package/dist/vectors/sqlite-vec.js +362 -0
- package/dist/vectors/sqlite-vec.js.map +1 -0
- package/package.json +64 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync Engine
|
|
3
|
+
*
|
|
4
|
+
* Synchronizes data from external readonly sources to LokiJS collections.
|
|
5
|
+
*/
|
|
6
|
+
import type { TableMapping, SyncResult, ImportOptions, PluginLogger } from '../types.js';
|
|
7
|
+
import type { SourceManager } from './source-manager.js';
|
|
8
|
+
/**
|
|
9
|
+
* LokiJS collection interface (minimal)
|
|
10
|
+
*/
|
|
11
|
+
interface LokiCollection<T = Record<string, unknown>> {
|
|
12
|
+
insert(doc: T): T;
|
|
13
|
+
update(doc: T): T;
|
|
14
|
+
findOne(query: object): T | null;
|
|
15
|
+
find(query?: object): T[];
|
|
16
|
+
remove(doc: T | T[]): void;
|
|
17
|
+
clear(): void;
|
|
18
|
+
count(): number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* LokiJS database interface (minimal)
|
|
22
|
+
*/
|
|
23
|
+
interface LokiDatabase {
|
|
24
|
+
getCollection<T>(name: string): LokiCollection<T> | null;
|
|
25
|
+
addCollection<T>(name: string, options?: object): LokiCollection<T>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Sync job
|
|
29
|
+
*/
|
|
30
|
+
interface SyncJob {
|
|
31
|
+
source: string;
|
|
32
|
+
table: string;
|
|
33
|
+
collection: string;
|
|
34
|
+
intervalMinutes: number;
|
|
35
|
+
lastRun?: number;
|
|
36
|
+
nextRun?: number;
|
|
37
|
+
running: boolean;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Sync engine interface
|
|
41
|
+
*/
|
|
42
|
+
export interface SyncEngine {
|
|
43
|
+
/** Import data from a source */
|
|
44
|
+
import(options: ImportOptions): Promise<SyncResult>;
|
|
45
|
+
/** Import a specific table */
|
|
46
|
+
importTable(source: string, table: string, mapping: TableMapping, fullRefresh?: boolean): Promise<SyncResult>;
|
|
47
|
+
/** Sync all configured sources */
|
|
48
|
+
syncAll(): Promise<SyncResult[]>;
|
|
49
|
+
/** Sync a specific source */
|
|
50
|
+
sync(source: string): Promise<SyncResult[]>;
|
|
51
|
+
/** Start auto-sync for a source */
|
|
52
|
+
startAutoSync(source: string): void;
|
|
53
|
+
/** Stop auto-sync for a source */
|
|
54
|
+
stopAutoSync(source: string): void;
|
|
55
|
+
/** Get sync jobs */
|
|
56
|
+
getJobs(): SyncJob[];
|
|
57
|
+
/** Get last sync time */
|
|
58
|
+
getLastSync(source: string): number | undefined;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Create sync engine
|
|
62
|
+
*/
|
|
63
|
+
export declare function createSyncEngine(sourceManager: SourceManager, lokiDb: LokiDatabase, logger: PluginLogger, options?: {
|
|
64
|
+
batchSize?: number;
|
|
65
|
+
defaultSyncInterval?: number;
|
|
66
|
+
}): SyncEngine;
|
|
67
|
+
export {};
|
|
68
|
+
//# sourceMappingURL=sync-engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sync-engine.d.ts","sourceRoot":"","sources":["../../src/federation/sync-engine.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,aAAa,EACb,YAAY,EACb,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD;;GAEG;AACH,UAAU,cAAc,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAClD,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IAClB,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IAClB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC;IACjC,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;IAC1B,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;IAC3B,KAAK,IAAI,IAAI,CAAC;IACd,KAAK,IAAI,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,UAAU,YAAY;IACpB,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IACzD,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,UAAU,OAAO;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,gCAAgC;IAChC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEpD,8BAA8B;IAC9B,WAAW,CACT,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,YAAY,EACrB,WAAW,CAAC,EAAE,OAAO,GACpB,OAAO,CAAC,UAAU,CAAC,CAAC;IAEvB,kCAAkC;IAClC,OAAO,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAEjC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAE5C,mCAAmC;IACnC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC,kCAAkC;IAClC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEnC,oBAAoB;IACpB,OAAO,IAAI,OAAO,EAAE,CAAC;IAErB,yBAAyB;IACzB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CACjD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,aAAa,EAAE,aAAa,EAC5B,MAAM,EAAE,YAAY,EACpB,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE;IACR,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,GACA,UAAU,CAyUZ"}
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync Engine
|
|
3
|
+
*
|
|
4
|
+
* Synchronizes data from external readonly sources to LokiJS collections.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Create sync engine
|
|
8
|
+
*/
|
|
9
|
+
export function createSyncEngine(sourceManager, lokiDb, logger, options) {
|
|
10
|
+
const batchSize = options?.batchSize ?? 1000;
|
|
11
|
+
const defaultSyncInterval = options?.defaultSyncInterval ?? 60; // minutes
|
|
12
|
+
const syncJobs = new Map();
|
|
13
|
+
const syncIntervals = new Map();
|
|
14
|
+
const lastSyncTimes = new Map();
|
|
15
|
+
/**
|
|
16
|
+
* Import data from a source
|
|
17
|
+
*/
|
|
18
|
+
async function importData(importOptions) {
|
|
19
|
+
const { source, table, fullRefresh = false } = importOptions;
|
|
20
|
+
const managedSource = sourceManager.getSource(source);
|
|
21
|
+
if (!managedSource) {
|
|
22
|
+
return {
|
|
23
|
+
success: false,
|
|
24
|
+
source,
|
|
25
|
+
table: table ?? '*',
|
|
26
|
+
collection: '',
|
|
27
|
+
inserted: 0,
|
|
28
|
+
updated: 0,
|
|
29
|
+
deleted: 0,
|
|
30
|
+
errors: [`Source '${source}' not found`],
|
|
31
|
+
duration: 0,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
const config = managedSource.config;
|
|
35
|
+
// If specific table, import just that
|
|
36
|
+
if (table && config.tables[table]) {
|
|
37
|
+
return importTable(source, table, config.tables[table], fullRefresh);
|
|
38
|
+
}
|
|
39
|
+
// Import all tables
|
|
40
|
+
const results = [];
|
|
41
|
+
for (const [tableName, mapping] of Object.entries(config.tables)) {
|
|
42
|
+
if (!table || tableName === table) {
|
|
43
|
+
const result = await importTable(source, tableName, mapping, fullRefresh);
|
|
44
|
+
results.push(result);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// Aggregate results
|
|
48
|
+
return {
|
|
49
|
+
success: results.every((r) => r.success),
|
|
50
|
+
source,
|
|
51
|
+
table: table ?? '*',
|
|
52
|
+
collection: results.map((r) => r.collection).join(', '),
|
|
53
|
+
inserted: results.reduce((sum, r) => sum + r.inserted, 0),
|
|
54
|
+
updated: results.reduce((sum, r) => sum + r.updated, 0),
|
|
55
|
+
deleted: results.reduce((sum, r) => sum + r.deleted, 0),
|
|
56
|
+
errors: results.flatMap((r) => r.errors),
|
|
57
|
+
duration: results.reduce((sum, r) => sum + r.duration, 0),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Import a specific table
|
|
62
|
+
*/
|
|
63
|
+
async function importTable(source, table, mapping, fullRefresh = false) {
|
|
64
|
+
const startTime = Date.now();
|
|
65
|
+
const result = {
|
|
66
|
+
success: false,
|
|
67
|
+
source,
|
|
68
|
+
table,
|
|
69
|
+
collection: mapping.collection,
|
|
70
|
+
inserted: 0,
|
|
71
|
+
updated: 0,
|
|
72
|
+
deleted: 0,
|
|
73
|
+
errors: [],
|
|
74
|
+
duration: 0,
|
|
75
|
+
};
|
|
76
|
+
const provider = sourceManager.getProvider(source);
|
|
77
|
+
if (!provider || !provider.isConnected()) {
|
|
78
|
+
result.errors.push(`Source '${source}' not connected`);
|
|
79
|
+
result.duration = Date.now() - startTime;
|
|
80
|
+
return result;
|
|
81
|
+
}
|
|
82
|
+
try {
|
|
83
|
+
// Get or create LokiJS collection
|
|
84
|
+
let collection = lokiDb.getCollection(mapping.collection);
|
|
85
|
+
if (!collection) {
|
|
86
|
+
collection = lokiDb.addCollection(mapping.collection, {
|
|
87
|
+
indices: [mapping.primaryKey ?? 'id'],
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
// Full refresh: clear collection first
|
|
91
|
+
if (fullRefresh) {
|
|
92
|
+
const existingCount = collection.count();
|
|
93
|
+
collection.clear();
|
|
94
|
+
result.deleted = existingCount;
|
|
95
|
+
}
|
|
96
|
+
// Build query
|
|
97
|
+
const columns = mapping.columns?.join(', ') ?? '*';
|
|
98
|
+
const whereClause = mapping.where ? ` WHERE ${mapping.where}` : '';
|
|
99
|
+
const sql = `SELECT ${columns} FROM ${provider.escapeIdentifier(table)}${whereClause}`;
|
|
100
|
+
// Fetch data in batches
|
|
101
|
+
let offset = 0;
|
|
102
|
+
let hasMore = true;
|
|
103
|
+
while (hasMore) {
|
|
104
|
+
const batchSql = `${sql} LIMIT ${batchSize} OFFSET ${offset}`;
|
|
105
|
+
const batchResult = await provider.query(batchSql);
|
|
106
|
+
if (batchResult.rows.length === 0) {
|
|
107
|
+
hasMore = false;
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
for (const row of batchResult.rows) {
|
|
111
|
+
// Apply transform if provided
|
|
112
|
+
const doc = mapping.transform ? mapping.transform(row) : row;
|
|
113
|
+
// Get primary key value
|
|
114
|
+
const pkField = mapping.primaryKey ?? 'id';
|
|
115
|
+
const pkValue = doc[pkField];
|
|
116
|
+
if (pkValue === undefined) {
|
|
117
|
+
result.errors.push(`Row missing primary key '${pkField}'`);
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
// Check if exists
|
|
121
|
+
const existing = collection.findOne({ [pkField]: pkValue });
|
|
122
|
+
if (existing) {
|
|
123
|
+
// Update
|
|
124
|
+
Object.assign(existing, doc);
|
|
125
|
+
collection.update(existing);
|
|
126
|
+
result.updated++;
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
// Insert
|
|
130
|
+
collection.insert(doc);
|
|
131
|
+
result.inserted++;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
offset += batchSize;
|
|
135
|
+
if (batchResult.rows.length < batchSize) {
|
|
136
|
+
hasMore = false;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
result.success = true;
|
|
140
|
+
lastSyncTimes.set(`${source}:${table}`, Date.now());
|
|
141
|
+
logger.info(`Imported ${result.inserted} inserted, ${result.updated} updated from ${source}.${table} to ${mapping.collection}`);
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
145
|
+
result.errors.push(message);
|
|
146
|
+
logger.error(`Import failed for ${source}.${table}: ${message}`);
|
|
147
|
+
}
|
|
148
|
+
result.duration = Date.now() - startTime;
|
|
149
|
+
return result;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Sync all sources
|
|
153
|
+
*/
|
|
154
|
+
async function syncAll() {
|
|
155
|
+
const results = [];
|
|
156
|
+
for (const source of sourceManager.getAllSources()) {
|
|
157
|
+
if (source.config.autoSync) {
|
|
158
|
+
const sourceResults = await sync(source.name);
|
|
159
|
+
results.push(...sourceResults);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return results;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Sync a specific source
|
|
166
|
+
*/
|
|
167
|
+
async function sync(sourceName) {
|
|
168
|
+
const source = sourceManager.getSource(sourceName);
|
|
169
|
+
if (!source) {
|
|
170
|
+
return [{
|
|
171
|
+
success: false,
|
|
172
|
+
source: sourceName,
|
|
173
|
+
table: '*',
|
|
174
|
+
collection: '',
|
|
175
|
+
inserted: 0,
|
|
176
|
+
updated: 0,
|
|
177
|
+
deleted: 0,
|
|
178
|
+
errors: [`Source '${sourceName}' not found`],
|
|
179
|
+
duration: 0,
|
|
180
|
+
}];
|
|
181
|
+
}
|
|
182
|
+
const results = [];
|
|
183
|
+
for (const [tableName, mapping] of Object.entries(source.config.tables)) {
|
|
184
|
+
const result = await importTable(sourceName, tableName, mapping, false);
|
|
185
|
+
results.push(result);
|
|
186
|
+
}
|
|
187
|
+
return results;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Start auto-sync for a source
|
|
191
|
+
*/
|
|
192
|
+
function startAutoSync(sourceName) {
|
|
193
|
+
const source = sourceManager.getSource(sourceName);
|
|
194
|
+
if (!source) {
|
|
195
|
+
logger.warn(`Cannot start auto-sync: source '${sourceName}' not found`);
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
// Stop existing sync if running
|
|
199
|
+
stopAutoSync(sourceName);
|
|
200
|
+
// Create sync jobs for each table
|
|
201
|
+
for (const [tableName, mapping] of Object.entries(source.config.tables)) {
|
|
202
|
+
const jobKey = `${sourceName}:${tableName}`;
|
|
203
|
+
const interval = mapping.syncInterval ?? defaultSyncInterval;
|
|
204
|
+
syncJobs.set(jobKey, {
|
|
205
|
+
source: sourceName,
|
|
206
|
+
table: tableName,
|
|
207
|
+
collection: mapping.collection,
|
|
208
|
+
intervalMinutes: interval,
|
|
209
|
+
running: false,
|
|
210
|
+
nextRun: Date.now() + interval * 60 * 1000,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
// Start interval
|
|
214
|
+
const interval = setInterval(async () => {
|
|
215
|
+
const now = Date.now();
|
|
216
|
+
for (const [jobKey, job] of syncJobs) {
|
|
217
|
+
if (!job.running && job.nextRun && now >= job.nextRun) {
|
|
218
|
+
job.running = true;
|
|
219
|
+
try {
|
|
220
|
+
const mapping = source.config.tables[job.table];
|
|
221
|
+
if (mapping) {
|
|
222
|
+
await importTable(job.source, job.table, mapping, false);
|
|
223
|
+
job.lastRun = Date.now();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
logger.error(`Auto-sync failed for ${jobKey}: ${error}`);
|
|
228
|
+
}
|
|
229
|
+
finally {
|
|
230
|
+
job.running = false;
|
|
231
|
+
job.nextRun = Date.now() + job.intervalMinutes * 60 * 1000;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}, 60 * 1000); // Check every minute
|
|
236
|
+
syncIntervals.set(sourceName, interval);
|
|
237
|
+
logger.info(`Started auto-sync for source: ${sourceName}`);
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Stop auto-sync for a source
|
|
241
|
+
*/
|
|
242
|
+
function stopAutoSync(sourceName) {
|
|
243
|
+
const interval = syncIntervals.get(sourceName);
|
|
244
|
+
if (interval) {
|
|
245
|
+
clearInterval(interval);
|
|
246
|
+
syncIntervals.delete(sourceName);
|
|
247
|
+
}
|
|
248
|
+
// Remove jobs for this source
|
|
249
|
+
for (const jobKey of syncJobs.keys()) {
|
|
250
|
+
if (jobKey.startsWith(`${sourceName}:`)) {
|
|
251
|
+
syncJobs.delete(jobKey);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
logger.info(`Stopped auto-sync for source: ${sourceName}`);
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Get sync jobs
|
|
258
|
+
*/
|
|
259
|
+
function getJobs() {
|
|
260
|
+
return Array.from(syncJobs.values());
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Get last sync time
|
|
264
|
+
*/
|
|
265
|
+
function getLastSync(source) {
|
|
266
|
+
// Find most recent sync for this source
|
|
267
|
+
let lastSync;
|
|
268
|
+
for (const [key, time] of lastSyncTimes) {
|
|
269
|
+
if (key.startsWith(`${source}:`)) {
|
|
270
|
+
if (!lastSync || time > lastSync) {
|
|
271
|
+
lastSync = time;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return lastSync;
|
|
276
|
+
}
|
|
277
|
+
return {
|
|
278
|
+
import: importData,
|
|
279
|
+
importTable,
|
|
280
|
+
syncAll,
|
|
281
|
+
sync,
|
|
282
|
+
startAutoSync,
|
|
283
|
+
stopAutoSync,
|
|
284
|
+
getJobs,
|
|
285
|
+
getLastSync,
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
//# sourceMappingURL=sync-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sync-engine.js","sourceRoot":"","sources":["../../src/federation/sync-engine.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA8EH;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,aAA4B,EAC5B,MAAoB,EACpB,MAAoB,EACpB,OAGC;IAED,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;IAC7C,MAAM,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,EAAE,CAAC,CAAC,UAAU;IAE1E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,EAA0B,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEhD;;OAEG;IACH,KAAK,UAAU,UAAU,CAAC,aAA4B;QACpD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,KAAK,EAAE,GAAG,aAAa,CAAC;QAE7D,MAAM,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,MAAM;gBACN,KAAK,EAAE,KAAK,IAAI,GAAG;gBACnB,UAAU,EAAE,EAAE;gBACd,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC,WAAW,MAAM,aAAa,CAAC;gBACxC,QAAQ,EAAE,CAAC;aACZ,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QAEpC,sCAAsC;QACtC,IAAI,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC,CAAC;QACvE,CAAC;QAED,oBAAoB;QACpB,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,KAAK,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBAClC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;YACxC,MAAM;YACN,KAAK,EAAE,KAAK,IAAI,GAAG;YACnB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACvD,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACzD,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACvD,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACvD,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACxC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,UAAU,WAAW,CACxB,MAAc,EACd,KAAa,EACb,OAAqB,EACrB,WAAW,GAAG,KAAK;QAEnB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAe;YACzB,OAAO,EAAE,KAAK;YACd,MAAM;YACN,KAAK;YACL,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,CAAC;SACZ,CAAC;QAEF,MAAM,QAAQ,GAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,MAAM,iBAAiB,CAAC,CAAC;YACvD,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,CAAC;YACH,kCAAkC;YAClC,IAAI,UAAU,GAAG,MAAM,CAAC,aAAa,CAA0B,OAAO,CAAC,UAAU,CAAC,CAAC;YACnF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,UAAU,GAAG,MAAM,CAAC,aAAa,CAA0B,OAAO,CAAC,UAAU,EAAE;oBAC7E,OAAO,EAAE,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;YAED,uCAAuC;YACvC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,aAAa,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC;gBACzC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,OAAO,GAAG,aAAa,CAAC;YACjC,CAAC;YAED,cAAc;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YACnD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,MAAM,GAAG,GAAG,UAAU,OAAO,SAAS,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,WAAW,EAAE,CAAC;YAEvF,wBAAwB;YACxB,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,OAAO,GAAG,IAAI,CAAC;YAEnB,OAAO,OAAO,EAAE,CAAC;gBACf,MAAM,QAAQ,GAAG,GAAG,GAAG,UAAU,SAAS,WAAW,MAAM,EAAE,CAAC;gBAC9D,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CAA0B,QAAQ,CAAC,CAAC;gBAE5E,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAClC,OAAO,GAAG,KAAK,CAAC;oBAChB,MAAM;gBACR,CAAC;gBAED,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;oBACnC,8BAA8B;oBAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;oBAE7D,wBAAwB;oBACxB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;oBAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;oBAE7B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;wBAC1B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,OAAO,GAAG,CAAC,CAAC;wBAC3D,SAAS;oBACX,CAAC;oBAED,kBAAkB;oBAClB,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;oBAE5D,IAAI,QAAQ,EAAE,CAAC;wBACb,SAAS;wBACT,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;wBAC7B,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;wBAC5B,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,CAAC;yBAAM,CAAC;wBACN,SAAS;wBACT,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBACvB,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAED,MAAM,IAAI,SAAS,CAAC;gBAEpB,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;oBACxC,OAAO,GAAG,KAAK,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,aAAa,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAEpD,MAAM,CAAC,IAAI,CACT,YAAY,MAAM,CAAC,QAAQ,cAAc,MAAM,CAAC,OAAO,iBAAiB,MAAM,IAAI,KAAK,OAAO,OAAO,CAAC,UAAU,EAAE,CACnH,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,qBAAqB,MAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACzC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,UAAU,OAAO;QACpB,MAAM,OAAO,GAAiB,EAAE,CAAC;QAEjC,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,aAAa,EAAE,EAAE,CAAC;YACnD,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC3B,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC9C,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,UAAU,IAAI,CAAC,UAAkB;QACpC,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC;oBACN,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,UAAU;oBAClB,KAAK,EAAE,GAAG;oBACV,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,CAAC;oBACX,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,CAAC,WAAW,UAAU,aAAa,CAAC;oBAC5C,QAAQ,EAAE,CAAC;iBACZ,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAiB,EAAE,CAAC;QAEjC,KAAK,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACxE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,SAAS,aAAa,CAAC,UAAkB;QACvC,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,mCAAmC,UAAU,aAAa,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QAED,gCAAgC;QAChC,YAAY,CAAC,UAAU,CAAC,CAAC;QAEzB,kCAAkC;QAClC,KAAK,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACxE,MAAM,MAAM,GAAG,GAAG,UAAU,IAAI,SAAS,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,IAAI,mBAAmB,CAAC;YAE7D,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;gBACnB,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,eAAe,EAAE,QAAQ;gBACzB,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,EAAE,GAAG,IAAI;aAC3C,CAAC,CAAC;QACL,CAAC;QAED,iBAAiB;QACjB,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEvB,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;oBACtD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;oBAEnB,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;wBAChD,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;4BACzD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBAC3B,CAAC;oBACH,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,KAAK,CAAC,wBAAwB,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;oBAC3D,CAAC;4BAAS,CAAC;wBACT,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;wBACpB,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC;oBAC7D,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,qBAAqB;QAEpC,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,iCAAiC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,SAAS,YAAY,CAAC,UAAkB;QACtC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,QAAQ,EAAE,CAAC;YACb,aAAa,CAAC,QAAQ,CAAC,CAAC;YACxB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAED,8BAA8B;QAC9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;gBACxC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,iCAAiC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,SAAS,OAAO;QACd,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS,WAAW,CAAC,MAAc;QACjC,wCAAwC;QACxC,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,aAAa,EAAE,CAAC;YACxC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjC,IAAI,CAAC,QAAQ,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACjC,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,WAAW;QACX,OAAO;QACP,IAAI;QACJ,aAAa;QACb,YAAY;QACZ,OAAO;QACP,WAAW;KACZ,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LokiCMS SQL Plugin
|
|
3
|
+
*
|
|
4
|
+
* Universal SQL adapter with multi-database support, data federation,
|
|
5
|
+
* split storage, and vector capabilities across all providers.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import sqlPlugin from 'lokicms-plugin-sql';
|
|
10
|
+
*
|
|
11
|
+
* const cms = createLokiCMS({
|
|
12
|
+
* plugins: [sqlPlugin],
|
|
13
|
+
* sql: {
|
|
14
|
+
* primary: {
|
|
15
|
+
* provider: 'postgresql',
|
|
16
|
+
* mode: 'readwrite',
|
|
17
|
+
* connection: {
|
|
18
|
+
* connectionString: process.env.DATABASE_URL,
|
|
19
|
+
* },
|
|
20
|
+
* },
|
|
21
|
+
* sources: [
|
|
22
|
+
* {
|
|
23
|
+
* name: 'legacy',
|
|
24
|
+
* provider: 'mysql',
|
|
25
|
+
* mode: 'readonly',
|
|
26
|
+
* connection: {
|
|
27
|
+
* host: 'legacy-db.example.com',
|
|
28
|
+
* database: 'legacy_data',
|
|
29
|
+
* user: 'readonly',
|
|
30
|
+
* password: process.env.LEGACY_DB_PASSWORD,
|
|
31
|
+
* },
|
|
32
|
+
* tables: {
|
|
33
|
+
* users: { collection: 'legacyUsers', syncInterval: 60 },
|
|
34
|
+
* products: { collection: 'legacyProducts', syncInterval: 30 },
|
|
35
|
+
* },
|
|
36
|
+
* autoSync: true,
|
|
37
|
+
* },
|
|
38
|
+
* ],
|
|
39
|
+
* vectors: {
|
|
40
|
+
* provider: 'postgresql',
|
|
41
|
+
* dimensions: 384,
|
|
42
|
+
* enableIndex: true,
|
|
43
|
+
* },
|
|
44
|
+
* cache: {
|
|
45
|
+
* enabled: true,
|
|
46
|
+
* ttlMinutes: 60,
|
|
47
|
+
* },
|
|
48
|
+
* },
|
|
49
|
+
* });
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
export { default } from './plugin.js';
|
|
53
|
+
export { getSourceManager, getSyncEngine, getVectorAdapter } from './plugin.js';
|
|
54
|
+
export { createProvider, getSupportedProviders, isProviderSupported } from './providers/index.js';
|
|
55
|
+
export type { SQLProvider, BaseProviderOptions } from './providers/index.js';
|
|
56
|
+
export { BaseProvider } from './providers/index.js';
|
|
57
|
+
export { PostgreSQLProvider } from './providers/postgresql.js';
|
|
58
|
+
export { MySQLProvider } from './providers/mysql.js';
|
|
59
|
+
export { MariaDBProvider } from './providers/mariadb.js';
|
|
60
|
+
export { SQLiteProvider } from './providers/sqlite.js';
|
|
61
|
+
export { createSourceManager } from './federation/source-manager.js';
|
|
62
|
+
export type { SourceManager, ManagedSource } from './federation/source-manager.js';
|
|
63
|
+
export { createSyncEngine } from './federation/sync-engine.js';
|
|
64
|
+
export type { SyncEngine } from './federation/sync-engine.js';
|
|
65
|
+
export { createCacheLayer } from './federation/cache-layer.js';
|
|
66
|
+
export type { CacheLayer } from './federation/cache-layer.js';
|
|
67
|
+
export { createVectorAdapter, createLokiVectorAdapter, getVectorProviderInfo } from './vectors/index.js';
|
|
68
|
+
export type { VectorAdapter, VectorAdapterOptions } from './vectors/index.js';
|
|
69
|
+
export { PgVectorAdapter } from './vectors/pgvector.js';
|
|
70
|
+
export { MySQLVectorAdapter } from './vectors/mysql-vector.js';
|
|
71
|
+
export { MariaDBVectorAdapter } from './vectors/mariadb-vector.js';
|
|
72
|
+
export { SQLiteVecAdapter } from './vectors/sqlite-vec.js';
|
|
73
|
+
export { LokiVectorAdapter } from './vectors/lokijs-vector.js';
|
|
74
|
+
export { createQueryBuilder } from './query/builder.js';
|
|
75
|
+
export type { QueryBuilder, LokiQuery, QueryOptions, BuiltQuery } from './query/builder.js';
|
|
76
|
+
export { createRowTransformer, createAutoMapping, createEntryTransformer } from './query/transformer.js';
|
|
77
|
+
export type { RowTransformer, SchemaMapping, FieldMapping } from './query/transformer.js';
|
|
78
|
+
export type { SQLProviderType, VectorProviderType, ConnectionMode, PostgreSQLConnectionConfig, MySQLConnectionConfig, SQLiteConnectionConfig, ConnectionConfig, SQLPluginConfig, SourceConfig, VectorConfig, CacheConfig, TableMapping, ProviderCapabilities, ProviderStatus, QueryResult, VectorEntry, VectorSearchOptions, VectorSearchResult, SyncResult, ImportOptions, CacheEntry, CacheStats, DatabaseSchema, TableInfo, ColumnInfo, PluginDefinition, PluginAPI, PluginLogger, PluginHooks, McpTools, McpToolDefinition, SQLErrorCode, } from './types.js';
|
|
79
|
+
export { SQL_PROVIDERS, VECTOR_PROVIDERS, DEFAULT_TABLE_PREFIX, DEFAULT_VECTOR_DIMENSIONS, DEFAULT_CACHE_TTL, SQLError, } from './types.js';
|
|
80
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGhF,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAClG,YAAY,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,YAAY,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,YAAY,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAG9D,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACzG,YAAY,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAG/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACzG,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAG1F,YAAY,EAEV,eAAe,EACf,kBAAkB,EAClB,cAAc,EAGd,0BAA0B,EAC1B,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAGhB,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,YAAY,EAGZ,oBAAoB,EACpB,cAAc,EACd,WAAW,EAGX,WAAW,EACX,mBAAmB,EACnB,kBAAkB,EAGlB,UAAU,EACV,aAAa,EAGb,UAAU,EACV,UAAU,EAGV,cAAc,EACd,SAAS,EACT,UAAU,EAGV,gBAAgB,EAChB,SAAS,EACT,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,iBAAiB,EAGjB,YAAY,GACb,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,oBAAoB,EACpB,yBAAyB,EACzB,iBAAiB,EACjB,QAAQ,GACT,MAAM,YAAY,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LokiCMS SQL Plugin
|
|
3
|
+
*
|
|
4
|
+
* Universal SQL adapter with multi-database support, data federation,
|
|
5
|
+
* split storage, and vector capabilities across all providers.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import sqlPlugin from 'lokicms-plugin-sql';
|
|
10
|
+
*
|
|
11
|
+
* const cms = createLokiCMS({
|
|
12
|
+
* plugins: [sqlPlugin],
|
|
13
|
+
* sql: {
|
|
14
|
+
* primary: {
|
|
15
|
+
* provider: 'postgresql',
|
|
16
|
+
* mode: 'readwrite',
|
|
17
|
+
* connection: {
|
|
18
|
+
* connectionString: process.env.DATABASE_URL,
|
|
19
|
+
* },
|
|
20
|
+
* },
|
|
21
|
+
* sources: [
|
|
22
|
+
* {
|
|
23
|
+
* name: 'legacy',
|
|
24
|
+
* provider: 'mysql',
|
|
25
|
+
* mode: 'readonly',
|
|
26
|
+
* connection: {
|
|
27
|
+
* host: 'legacy-db.example.com',
|
|
28
|
+
* database: 'legacy_data',
|
|
29
|
+
* user: 'readonly',
|
|
30
|
+
* password: process.env.LEGACY_DB_PASSWORD,
|
|
31
|
+
* },
|
|
32
|
+
* tables: {
|
|
33
|
+
* users: { collection: 'legacyUsers', syncInterval: 60 },
|
|
34
|
+
* products: { collection: 'legacyProducts', syncInterval: 30 },
|
|
35
|
+
* },
|
|
36
|
+
* autoSync: true,
|
|
37
|
+
* },
|
|
38
|
+
* ],
|
|
39
|
+
* vectors: {
|
|
40
|
+
* provider: 'postgresql',
|
|
41
|
+
* dimensions: 384,
|
|
42
|
+
* enableIndex: true,
|
|
43
|
+
* },
|
|
44
|
+
* cache: {
|
|
45
|
+
* enabled: true,
|
|
46
|
+
* ttlMinutes: 60,
|
|
47
|
+
* },
|
|
48
|
+
* },
|
|
49
|
+
* });
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
// Default plugin export
|
|
53
|
+
export { default } from './plugin.js';
|
|
54
|
+
// Plugin access
|
|
55
|
+
export { getSourceManager, getSyncEngine, getVectorAdapter } from './plugin.js';
|
|
56
|
+
// Providers
|
|
57
|
+
export { createProvider, getSupportedProviders, isProviderSupported } from './providers/index.js';
|
|
58
|
+
export { BaseProvider } from './providers/index.js';
|
|
59
|
+
export { PostgreSQLProvider } from './providers/postgresql.js';
|
|
60
|
+
export { MySQLProvider } from './providers/mysql.js';
|
|
61
|
+
export { MariaDBProvider } from './providers/mariadb.js';
|
|
62
|
+
export { SQLiteProvider } from './providers/sqlite.js';
|
|
63
|
+
// Federation
|
|
64
|
+
export { createSourceManager } from './federation/source-manager.js';
|
|
65
|
+
export { createSyncEngine } from './federation/sync-engine.js';
|
|
66
|
+
export { createCacheLayer } from './federation/cache-layer.js';
|
|
67
|
+
// Vectors
|
|
68
|
+
export { createVectorAdapter, createLokiVectorAdapter, getVectorProviderInfo } from './vectors/index.js';
|
|
69
|
+
export { PgVectorAdapter } from './vectors/pgvector.js';
|
|
70
|
+
export { MySQLVectorAdapter } from './vectors/mysql-vector.js';
|
|
71
|
+
export { MariaDBVectorAdapter } from './vectors/mariadb-vector.js';
|
|
72
|
+
export { SQLiteVecAdapter } from './vectors/sqlite-vec.js';
|
|
73
|
+
export { LokiVectorAdapter } from './vectors/lokijs-vector.js';
|
|
74
|
+
// Query
|
|
75
|
+
export { createQueryBuilder } from './query/builder.js';
|
|
76
|
+
export { createRowTransformer, createAutoMapping, createEntryTransformer } from './query/transformer.js';
|
|
77
|
+
// Constants
|
|
78
|
+
export { SQL_PROVIDERS, VECTOR_PROVIDERS, DEFAULT_TABLE_PREFIX, DEFAULT_VECTOR_DIMENSIONS, DEFAULT_CACHE_TTL, SQLError, } from './types.js';
|
|
79
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAEH,wBAAwB;AACxB,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,gBAAgB;AAChB,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEhF,YAAY;AACZ,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAElG,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,aAAa;AACb,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAG/D,UAAU;AACV,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEzG,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAE/D,QAAQ;AACR,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AA0DzG,YAAY;AACZ,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,oBAAoB,EACpB,yBAAyB,EACzB,iBAAiB,EACjB,QAAQ,GACT,MAAM,YAAY,CAAC"}
|
package/dist/plugin.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LokiCMS SQL Plugin
|
|
3
|
+
*
|
|
4
|
+
* Universal SQL adapter with multi-database support, data federation,
|
|
5
|
+
* split storage, and vector capabilities.
|
|
6
|
+
*/
|
|
7
|
+
import type { PluginDefinition } from './types.js';
|
|
8
|
+
import { SourceManager } from './federation/source-manager.js';
|
|
9
|
+
import { SyncEngine } from './federation/sync-engine.js';
|
|
10
|
+
import { VectorAdapter } from './vectors/index.js';
|
|
11
|
+
/**
|
|
12
|
+
* Get source manager for code usage
|
|
13
|
+
*/
|
|
14
|
+
export declare function getSourceManager(): SourceManager;
|
|
15
|
+
/**
|
|
16
|
+
* Get sync engine for code usage
|
|
17
|
+
*/
|
|
18
|
+
export declare function getSyncEngine(): SyncEngine;
|
|
19
|
+
/**
|
|
20
|
+
* Get vector adapter for code usage
|
|
21
|
+
*/
|
|
22
|
+
export declare function getVectorAdapter(): VectorAdapter | null;
|
|
23
|
+
/**
|
|
24
|
+
* LokiCMS SQL Plugin Definition
|
|
25
|
+
*/
|
|
26
|
+
declare const plugin: PluginDefinition;
|
|
27
|
+
export default plugin;
|
|
28
|
+
//# sourceMappingURL=plugin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EACV,gBAAgB,EAQjB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAuB,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACpF,OAAO,EAAoB,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAE3E,OAAO,EAAuB,aAAa,EAAE,MAAM,oBAAoB,CAAC;AASxE;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,aAAa,CAKhD;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,UAAU,CAK1C;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,aAAa,GAAG,IAAI,CAEvD;AAED;;GAEG;AACH,QAAA,MAAM,MAAM,EAAE,gBAkFb,CAAC;AA6xBF,eAAe,MAAM,CAAC"}
|