@meaningfully/core 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/MetadataManager.d.ts +2 -14
  2. package/dist/MetadataManager.d.ts.map +1 -1
  3. package/dist/MetadataManager.js +35 -69
  4. package/dist/MetadataManager.js.map +1 -1
  5. package/package.json +2 -1
  6. package/src/MetadataManager.ts +42 -84
  7. package/src/__tests__/MetadataManager.test.ts +37 -37
package/dist/MetadataManager.d.ts CHANGED
@@ -1,19 +1,7 @@
1
1
  import type { DocumentSetMetadata, Settings } from './types/index.js';
2
+ import type { Knex } from 'knex';
2
3
  export declare abstract class MetadataManager {
3
- protected queries: {
4
- createDocumentSetsTable: string;
5
- createSettingsTable: string;
6
- insertDocumentSet: string;
7
- selectDocumentSet: string;
8
- selectDocumentSets: string;
9
- countDocumentSets: string;
10
- updateDocumentCount: string;
11
- deleteDocumentSet: string;
12
- selectSettings: string;
13
- upsertSettings: string;
14
- };
15
- protected abstract runQuery<T>(query: string, params?: any[]): Promise<T[]>;
16
- protected abstract runQuerySingle<T>(query: string, params?: any[]): Promise<T | null>;
4
+ protected abstract knex: Knex;
17
5
  protected abstract initializeDatabase(): Promise<void>;
18
6
  protected abstract close(): void;
19
7
  addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number>;
package/dist/MetadataManager.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"MetadataManager.d.ts","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEtE,8BAAsB,eAAe;IACnC,SAAS,CAAC,OAAO;;;;;;;;;;;MAmDf;IAEF,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAC3E,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IACtF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IACtD,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;IAE1B,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAUrF,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAoB1E,eAAe,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAwBtH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxE,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvD,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAehC,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CAMrE"}
1
+ {"version":3,"file":"MetadataManager.d.ts","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAEjC,8BAAsB,eAAe;IACnC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAE9B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IACtD,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;IAE1B,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAarF,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAgB1E,eAAe,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAuBtH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMxE,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvD,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAkBhC,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CAWrE"}
package/dist/MetadataManager.js CHANGED
@@ -1,67 +1,19 @@
1
1
  export class MetadataManager {
2
- queries = {
3
- /*
4
- Note: RETURNING on non-select/non-create statements is important for compatibility between SQLite and PostgreSQL.
5
- (Without it, better-sqlite would demand to use run() instead of all() or get(), which would break the abstraction.)
6
- */
7
- createDocumentSetsTable: `
8
- CREATE TABLE IF NOT EXISTS document_sets (
9
- set_id SERIAL PRIMARY KEY,
10
- name TEXT NOT NULL UNIQUE,
11
- upload_date TIMESTAMP NOT NULL,
12
- parameters TEXT NOT NULL,
13
- total_documents INTEGER NOT NULL DEFAULT 0
14
- );
15
- `,
16
- createSettingsTable: `
17
- CREATE TABLE IF NOT EXISTS meaningfully_settings (
18
- settings_id SERIAL PRIMARY KEY,
19
- settings TEXT NOT NULL
20
- );
21
- `,
22
- insertDocumentSet: `
23
- INSERT INTO document_sets (name, upload_date, parameters, total_documents)
24
- VALUES ($1, $2, $3, $4) RETURNING set_id
25
- `,
26
- selectDocumentSet: `
27
- SELECT * FROM document_sets WHERE set_id = $1
28
- `,
29
- selectDocumentSets: `
30
- SELECT * FROM document_sets ORDER BY upload_date DESC LIMIT $1 OFFSET $2
31
- `,
32
- countDocumentSets: `
33
- SELECT COUNT(*) as count FROM document_sets
34
- `,
35
- updateDocumentCount: `
36
- UPDATE document_sets SET total_documents = total_documents + $1 WHERE set_id = $2 RETURNING *
37
- `,
38
- deleteDocumentSet: `
39
- DELETE FROM document_sets WHERE set_id = $1 RETURNING *
40
- `,
41
- selectSettings: `
42
- SELECT * FROM meaningfully_settings WHERE settings_id = 1
43
- `,
44
- upsertSettings: `
45
- INSERT INTO meaningfully_settings (settings_id, settings)
46
- VALUES (1, $1)
47
- ON CONFLICT (settings_id) DO UPDATE SET settings = $2
48
- RETURNING *
49
- `
50
- // the two arguments $1 and $2 are identical, but, to work around a cross-compatibility bug in SQLite versus Postgresql,
51
- // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
52
- // they are specified separately.
53
- };
54
2
  async addDocumentSet(metadata) {
55
- const result = await this.runQuerySingle(this.queries.insertDocumentSet, [
56
- metadata.name,
57
- metadata.uploadDate.toISOString(),
58
- JSON.stringify(metadata.parameters),
59
- metadata.totalDocuments
60
- ]);
61
- return result?.set_id || 0;
3
+ const [result] = await this.knex('document_sets')
4
+ .insert({
5
+ name: metadata.name,
6
+ upload_date: metadata.uploadDate,
7
+ parameters: JSON.stringify(metadata.parameters),
8
+ total_documents: metadata.totalDocuments
9
+ })
10
+ .returning('set_id');
11
+ return typeof result === 'object' ? result.set_id : result;
62
12
  }
63
13
  async getDocumentSet(documentSetId) {
64
- const row = await this.runQuerySingle(this.queries.selectDocumentSet, [documentSetId]);
14
+ const row = await this.knex('document_sets')
15
+ .where('set_id', documentSetId)
16
+ .first();
65
17
  if (!row)
66
18
  return null;
67
19
  return {
@@ -74,9 +26,13 @@ export class MetadataManager {
74
26
  }
75
27
  async getDocumentSets(page = 1, pageSize = 10) {
76
28
  const offset = (page - 1) * pageSize;
77
- const totalCountRow = await this.runQuerySingle(this.queries.countDocumentSets);
78
- const totalCount = totalCountRow?.count || 0;
79
- const rows = await this.runQuery(this.queries.selectDocumentSets, [pageSize, offset]);
29
+ const totalCountRow = await this.knex('document_sets').count('* as count').first();
30
+ const totalCount = totalCountRow ? Number(totalCountRow.count) : 0;
31
+ const rows = await this.knex('document_sets')
32
+ .select('*')
33
+ .orderBy('upload_date', 'desc')
34
+ .limit(pageSize)
35
+ .offset(offset);
80
36
  const documents = rows.map((row) => ({
81
37
  documentSetId: row.set_id,
82
38
  name: row.name,
@@ -87,10 +43,14 @@ export class MetadataManager {
87
43
  return { documents, total: totalCount };
88
44
  }
89
45
  async updateDocumentCount(documentSetId, count) {
90
- await this.runQuery(this.queries.updateDocumentCount, [count, documentSetId]);
46
+ await this.knex('document_sets')
47
+ .where('set_id', documentSetId)
48
+ .increment('total_documents', count);
91
49
  }
92
50
  async deleteDocumentSet(documentSetId) {
93
- await this.runQuery(this.queries.deleteDocumentSet, [documentSetId]);
51
+ await this.knex('document_sets')
52
+ .where('set_id', documentSetId)
53
+ .delete();
94
54
  }
95
55
  async getSettings() {
96
56
  const DEFAULT_SETTINGS = {
@@ -102,13 +62,19 @@ export class MetadataManager {
102
62
  mistralApiKey: null,
103
63
  geminiApiKey: null,
104
64
  };
105
- const row = await this.runQuerySingle(this.queries.selectSettings);
65
+ const row = await this.knex('meaningfully_settings')
66
+ .where('settings_id', 1)
67
+ .first();
106
68
  return row ? { ...DEFAULT_SETTINGS, ...JSON.parse(row.settings) } : DEFAULT_SETTINGS;
107
69
  }
108
70
  async setSettings(settings) {
109
- // the JSON.stringify(settings) is repeated to work around a cross-compatibility bug in SQLite versus Postgresql
110
- // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
111
- await this.runQuery(this.queries.upsertSettings, [JSON.stringify(settings), JSON.stringify(settings)]);
71
+ await this.knex('meaningfully_settings')
72
+ .insert({
73
+ settings_id: 1,
74
+ settings: JSON.stringify(settings)
75
+ })
76
+ .onConflict('settings_id')
77
+ .merge();
112
78
  return { success: true };
113
79
  }
114
80
  }
package/dist/MetadataManager.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"MetadataManager.js","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAEA,MAAM,OAAgB,eAAe;IACzB,OAAO,GAAG;QAClB;;;UAGE;QACF,uBAAuB,EAAE;;;;;;;;KAQxB;QACD,mBAAmB,EAAE;;;;;KAKpB;QACD,iBAAiB,EAAE;;;KAGlB;QACD,iBAAiB,EAAE;;KAElB;QACD,kBAAkB,EAAE;;KAEnB;QACD,iBAAiB,EAAE;;KAElB;QACD,mBAAmB,EAAE;;KAEpB;QACD,iBAAiB,EAAE;;KAElB;QACD,cAAc,EAAE;;KAEf;QACD,cAAc,EAAE;;;;;KAKf;QACD,wHAAwH;QACxH,8HAA8H;QAC9H,iCAAiC;KAClC,CAAC;IAOF,KAAK,CAAC,cAAc,CAAC,QAAoD;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAqB,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE;YAC3F,QAAQ,CAAC,IAAI;YACb,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE;YACjC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACnC,QAAQ,CAAC,cAAc;SACxB,CAAC,CAAC;QACH,OAAO,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAMlC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpD,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC3D,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QACrC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,cAAc,CAAoB,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACnG,MAAM,UAAU,GAAG,aAAa,EAAE,KAAK,IAAI,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAM7B,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAExD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,aAAqB,EAAE,KAAa;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,gBAAgB,GAAa;YACjC,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,mBAAmB,EAAE,IAAI;YACzB,qBAAqB,EAAE,YAAY;YACnC,aAAa,EAAE,IAAI;YACnB,YAAY,EAAE,IAAI;SACnB,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAuB,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzF,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,gHAAgH;QAChH,8HAA8H;QAC9H,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}
1
+ {"version":3,"file":"MetadataManager.js","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAGA,MAAM,OAAgB,eAAe;IAMnC,KAAK,CAAC,cAAc,CAAC,QAAoD;QACvE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC9C,MAAM,CAAC;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YAC/C,eAAe,EAAE,QAAQ,CAAC,cAAc;SACzC,CAAC;aACD,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEvB,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aACzC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC;aAC9B,KAAK,EAAE,CAAC;QAEX,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC3D,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QAErC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC;QACnF,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC1C,MAAM,CAAC,GAAG,CAAC;aACX,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC;aAC9B,KAAK,CAAC,QAAQ,CAAC;aACf,MAAM,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;YACxC,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,aAAqB,EAAE,KAAa;QAC5D,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC7B,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC;aAC9B,SAAS,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC7B,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC;aAC9B,MAAM,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,gBAAgB,GAAa;YACjC,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,mBAAmB,EAAE,IAAI;YACzB,qBAAqB,EAAE,YAAY;YACnC,aAAa,EAAE,IAAI;YACnB,YAAY,EAAE,IAAI;SACnB,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC;aACjD,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;aACvB,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC;aACrC,MAAM,CAAC;YACN,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;SACnC,CAAC;aACD,UAAU,CAAC,aAAa,CAAC;aACzB,KAAK,EAAE,CAAC;QAEX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@meaningfully/core",
3
- "version": "0.1.6",
3
+ "version": "0.1.7",
4
4
  "description": "Core functionality for meaningfully semantic search",
5
5
  "repository": "https://github.com/jeremybmerrill/meaningfully-core",
6
6
  "license": "MIT",
@@ -29,6 +29,7 @@
29
29
  "@llamaindex/postgres": "^0.0.64",
30
30
  "@llamaindex/weaviate": "^0.0.36",
31
31
  "js-tiktoken": "^1.0.8",
32
+ "knex": "^3.1.0",
32
33
  "llamaindex": "^0.11.14",
33
34
  "lodash": "^4.17.21",
34
35
  "natural": "^8.1.0",
package/src/MetadataManager.ts CHANGED
@@ -1,82 +1,29 @@
1
1
  import type { DocumentSetMetadata, Settings } from './types/index.js';
2
+ import type { Knex } from 'knex';
2
3
 
3
4
  export abstract class MetadataManager {
4
- protected queries = {
5
- /*
6
- Note: RETURNING on non-select/non-create statements is important for compatibility between SQLite and PostgreSQL.
7
- (Without it, better-sqlite would demand to use run() instead of all() or get(), which would break the abstraction.)
8
- */
9
- createDocumentSetsTable: `
10
- CREATE TABLE IF NOT EXISTS document_sets (
11
- set_id SERIAL PRIMARY KEY,
12
- name TEXT NOT NULL UNIQUE,
13
- upload_date TIMESTAMP NOT NULL,
14
- parameters TEXT NOT NULL,
15
- total_documents INTEGER NOT NULL DEFAULT 0
16
- );
17
- `,
18
- createSettingsTable: `
19
- CREATE TABLE IF NOT EXISTS meaningfully_settings (
20
- settings_id SERIAL PRIMARY KEY,
21
- settings TEXT NOT NULL
22
- );
23
- `,
24
- insertDocumentSet: `
25
- INSERT INTO document_sets (name, upload_date, parameters, total_documents)
26
- VALUES ($1, $2, $3, $4) RETURNING set_id
27
- `,
28
- selectDocumentSet: `
29
- SELECT * FROM document_sets WHERE set_id = $1
30
- `,
31
- selectDocumentSets: `
32
- SELECT * FROM document_sets ORDER BY upload_date DESC LIMIT $1 OFFSET $2
33
- `,
34
- countDocumentSets: `
35
- SELECT COUNT(*) as count FROM document_sets
36
- `,
37
- updateDocumentCount: `
38
- UPDATE document_sets SET total_documents = total_documents + $1 WHERE set_id = $2 RETURNING *
39
- `,
40
- deleteDocumentSet: `
41
- DELETE FROM document_sets WHERE set_id = $1 RETURNING *
42
- `,
43
- selectSettings: `
44
- SELECT * FROM meaningfully_settings WHERE settings_id = 1
45
- `,
46
- upsertSettings: `
47
- INSERT INTO meaningfully_settings (settings_id, settings)
48
- VALUES (1, $1)
49
- ON CONFLICT (settings_id) DO UPDATE SET settings = $2
50
- RETURNING *
51
- `
52
- // the two arguments $1 and $2 are identical, but, to work around a cross-compatibility bug in SQLite versus Postgresql,
53
- // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
54
- // they are specified separately.
55
- };
5
+ protected abstract knex: Knex;
56
6
 
57
- protected abstract runQuery<T>(query: string, params?: any[]): Promise<T[]>;
58
- protected abstract runQuerySingle<T>(query: string, params?: any[]): Promise<T | null>;
59
7
  protected abstract initializeDatabase(): Promise<void>;
60
8
  protected abstract close(): void;
61
9
 
62
10
  async addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number> {
63
- const result = await this.runQuerySingle<{ set_id: number }>(this.queries.insertDocumentSet, [
64
- metadata.name,
65
- metadata.uploadDate.toISOString(),
66
- JSON.stringify(metadata.parameters),
67
- metadata.totalDocuments
68
- ]);
69
- return result?.set_id || 0;
11
+ const [result] = await this.knex('document_sets')
12
+ .insert({
13
+ name: metadata.name,
14
+ upload_date: metadata.uploadDate,
15
+ parameters: JSON.stringify(metadata.parameters),
16
+ total_documents: metadata.totalDocuments
17
+ })
18
+ .returning('set_id');
19
+
20
+ return typeof result === 'object' ? result.set_id : result;
70
21
  }
71
22
 
72
23
  async getDocumentSet(documentSetId: number): Promise<DocumentSetMetadata | null> {
73
- const row = await this.runQuerySingle<{
74
- set_id: number;
75
- name: string;
76
- upload_date: string;
77
- parameters: string;
78
- total_documents: number;
79
- }>(this.queries.selectDocumentSet, [documentSetId]);
24
+ const row = await this.knex('document_sets')
25
+ .where('set_id', documentSetId)
26
+ .first();
80
27
 
81
28
  if (!row) return null;
82
29
 
@@ -91,18 +38,17 @@ export abstract class MetadataManager {
91
38
 
92
39
  async getDocumentSets(page: number = 1, pageSize: number = 10): Promise<{ documents: DocumentSetMetadata[]; total: number }> {
93
40
  const offset = (page - 1) * pageSize;
94
- const totalCountRow = await this.runQuerySingle<{ count: number }>(this.queries.countDocumentSets);
95
- const totalCount = totalCountRow?.count || 0;
41
+
42
+ const totalCountRow = await this.knex('document_sets').count('* as count').first();
43
+ const totalCount = totalCountRow ? Number(totalCountRow.count) : 0;
96
44
 
97
- const rows = await this.runQuery<{
98
- set_id: number;
99
- name: string;
100
- upload_date: string;
101
- parameters: string;
102
- total_documents: number;
103
- }>(this.queries.selectDocumentSets, [pageSize, offset]);
45
+ const rows = await this.knex('document_sets')
46
+ .select('*')
47
+ .orderBy('upload_date', 'desc')
48
+ .limit(pageSize)
49
+ .offset(offset);
104
50
 
105
- const documents = rows.map((row) => ({
51
+ const documents = rows.map((row: any) => ({
106
52
  documentSetId: row.set_id,
107
53
  name: row.name,
108
54
  uploadDate: new Date(row.upload_date),
@@ -114,11 +60,15 @@ export abstract class MetadataManager {
114
60
  }
115
61
 
116
62
  async updateDocumentCount(documentSetId: number, count: number): Promise<void> {
117
- await this.runQuery(this.queries.updateDocumentCount, [count, documentSetId]);
63
+ await this.knex('document_sets')
64
+ .where('set_id', documentSetId)
65
+ .increment('total_documents', count);
118
66
  }
119
67
 
120
68
  async deleteDocumentSet(documentSetId: number): Promise<void> {
121
- await this.runQuery(this.queries.deleteDocumentSet, [documentSetId]);
69
+ await this.knex('document_sets')
70
+ .where('set_id', documentSetId)
71
+ .delete();
122
72
  }
123
73
 
124
74
  async getSettings(): Promise<Settings> {
@@ -132,14 +82,22 @@ export abstract class MetadataManager {
132
82
  geminiApiKey: null,
133
83
  };
134
84
 
135
- const row = await this.runQuerySingle<{ settings: string }>(this.queries.selectSettings);
85
+ const row = await this.knex('meaningfully_settings')
86
+ .where('settings_id', 1)
87
+ .first();
88
+
136
89
  return row ? { ...DEFAULT_SETTINGS, ...JSON.parse(row.settings) } : DEFAULT_SETTINGS;
137
90
  }
138
91
 
139
92
  async setSettings(settings: Settings): Promise<{ success: boolean }> {
140
- // the JSON.stringify(settings) is repeated to work around a cross-compatibility bug in SQLite versus Postgresql
141
- // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
142
- await this.runQuery(this.queries.upsertSettings, [JSON.stringify(settings), JSON.stringify(settings)]);
93
+ await this.knex('meaningfully_settings')
94
+ .insert({
95
+ settings_id: 1,
96
+ settings: JSON.stringify(settings)
97
+ })
98
+ .onConflict('settings_id')
99
+ .merge();
100
+
143
101
  return { success: true };
144
102
  }
145
103
  }
package/src/__tests__/MetadataManager.test.ts CHANGED
@@ -1,24 +1,47 @@
1
1
  import { MetadataManager } from '../MetadataManager.js';
2
2
  import { describe, it, expect, vi, beforeEach } from 'vitest';
3
+ import type { Knex } from 'knex';
3
4
 
4
5
  describe('MetadataManager', () => {
5
6
  let metadataManager: MetadataManager;
7
+ let mockKnex: any;
6
8
 
7
9
  beforeEach(() => {
10
+ // Create a mock Knex instance with chainable methods
11
+ mockKnex = {
12
+ schema: {
13
+ hasTable: vi.fn().mockResolvedValue(true),
14
+ createTable: vi.fn().mockReturnThis(),
15
+ },
16
+ insert: vi.fn().mockReturnThis(),
17
+ returning: vi.fn().mockResolvedValue([{ set_id: 1 }]),
18
+ where: vi.fn().mockReturnThis(),
19
+ first: vi.fn().mockResolvedValue(null),
20
+ select: vi.fn().mockReturnThis(),
21
+ orderBy: vi.fn().mockReturnThis(),
22
+ limit: vi.fn().mockReturnThis(),
23
+ offset: vi.fn().mockResolvedValue([]),
24
+ count: vi.fn().mockReturnThis(),
25
+ increment: vi.fn().mockResolvedValue(undefined),
26
+ delete: vi.fn().mockResolvedValue(undefined),
27
+ onConflict: vi.fn().mockReturnThis(),
28
+ merge: vi.fn().mockResolvedValue(undefined),
29
+ destroy: vi.fn().mockResolvedValue(undefined),
30
+ };
31
+
32
+ // Make mockKnex callable as a function that returns itself for chaining
33
+ const knexFn = vi.fn().mockReturnValue(mockKnex);
34
+ Object.assign(knexFn, mockKnex);
35
+
8
36
  metadataManager = new (class extends MetadataManager {
9
- protected async runQuery<T>(query: string, params?: any[]): Promise<T[]> {
10
- return [] as T[];
11
- }
12
- protected async runQuerySingle<T>(query: string, params?: any[]): Promise<T | null> {
13
- return null;
14
- }
37
+ protected knex = knexFn as any;
15
38
  protected async initializeDatabase(): Promise<void> {}
16
39
  protected close(): void {}
17
40
  })();
18
41
  });
19
42
 
20
43
  it('should add a document set and return its ID', async () => {
21
- vi.spyOn(metadataManager, 'runQuerySingle').mockResolvedValueOnce({ set_id: 1 });
44
+ mockKnex.returning.mockResolvedValueOnce([{ set_id: 1 }]);
22
45
 
23
46
  const documentSetId = await metadataManager.addDocumentSet({
24
47
  name: 'Test Set',
@@ -31,7 +54,7 @@ describe('MetadataManager', () => {
31
54
  });
32
55
 
33
56
  it('should retrieve a document set by ID', async () => {
34
- vi.spyOn(metadataManager, 'runQuerySingle').mockResolvedValueOnce({
57
+ mockKnex.first.mockResolvedValueOnce({
35
58
  set_id: 1,
36
59
  name: 'Test Set',
37
60
  upload_date: new Date().toISOString(),
@@ -51,23 +74,19 @@ describe('MetadataManager', () => {
51
74
  });
52
75
 
53
76
  it('should update the document count for a document set', async () => {
54
- const runQuerySpy = vi.spyOn(metadataManager, 'runQuery').mockResolvedValueOnce([]);
55
-
56
77
  await metadataManager.updateDocumentCount(1, 5);
57
78
 
58
- expect(runQuerySpy).toHaveBeenCalledWith(metadataManager.queries.updateDocumentCount, [5, 1]);
79
+ expect(mockKnex.increment).toHaveBeenCalledWith('total_documents', 5);
59
80
  });
60
81
 
61
82
  it('should delete a document set by ID', async () => {
62
- const runQuerySpy = vi.spyOn(metadataManager, 'runQuery').mockResolvedValueOnce([]);
63
-
64
83
  await metadataManager.deleteDocumentSet(1);
65
84
 
66
- expect(runQuerySpy).toHaveBeenCalledWith(metadataManager.queries.deleteDocumentSet, [1]);
85
+ expect(mockKnex.delete).toHaveBeenCalled();
67
86
  });
68
87
 
69
88
  it('should retrieve default settings if none exist', async () => {
70
- vi.spyOn(metadataManager, 'runQuerySingle').mockResolvedValueOnce(null);
89
+ mockKnex.first.mockResolvedValueOnce(null);
71
90
 
72
91
  const settings = await metadataManager.getSettings();
73
92
 
@@ -83,8 +102,6 @@ describe('MetadataManager', () => {
83
102
  });
84
103
 
85
104
  it('should update settings', async () => {
86
- const runQuerySpy = vi.spyOn(metadataManager, 'runQuery').mockResolvedValueOnce([]);
87
-
88
105
  const result = await metadataManager.setSettings({
89
106
  openAIKey: 'test-key',
90
107
  oLlamaBaseURL: 'http://localhost',
@@ -96,25 +113,8 @@ describe('MetadataManager', () => {
96
113
  });
97
114
 
98
115
  expect(result).toEqual({ success: true });
99
- expect(runQuerySpy).toHaveBeenCalledWith(metadataManager.queries.upsertSettings, [
100
- JSON.stringify({
101
- openAIKey: 'test-key',
102
- oLlamaBaseURL: 'http://localhost',
103
- azureOpenAIKey: 'azure-key',
104
- azureOpenAIEndpoint: 'http://azure.endpoint',
105
- azureOpenAIApiVersion: '2024-02-01',
106
- mistralApiKey: 'mistral-key',
107
- geminiApiKey: 'gemini-key',
108
- }),
109
- JSON.stringify({
110
- openAIKey: 'test-key',
111
- oLlamaBaseURL: 'http://localhost',
112
- azureOpenAIKey: 'azure-key',
113
- azureOpenAIEndpoint: 'http://azure.endpoint',
114
- azureOpenAIApiVersion: '2024-02-01',
115
- mistralApiKey: 'mistral-key',
116
- geminiApiKey: 'gemini-key',
117
- }),
118
- ]);
116
+ expect(mockKnex.insert).toHaveBeenCalled();
117
+ expect(mockKnex.onConflict).toHaveBeenCalledWith('settings_id');
118
+ expect(mockKnex.merge).toHaveBeenCalled();
119
119
  });
120
120
  });