@memberjunction/ai-vector-sync 3.4.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +255 -178
  2. package/dist/config.js +14 -20
  3. package/dist/config.js.map +1 -1
  4. package/dist/db/db.d.ts +1 -1
  5. package/dist/db/db.d.ts.map +1 -1
  6. package/dist/db/db.js +8 -33
  7. package/dist/db/db.js.map +1 -1
  8. package/dist/db/dbAI.d.ts +1 -1
  9. package/dist/db/dbAI.d.ts.map +1 -1
  10. package/dist/db/dbAI.js +8 -33
  11. package/dist/db/dbAI.js.map +1 -1
  12. package/dist/generic/EntityDocumenTemplateParserBase.d.ts +12 -0
  13. package/dist/generic/EntityDocumenTemplateParserBase.d.ts.map +1 -1
  14. package/dist/generic/EntityDocumenTemplateParserBase.js +29 -12
  15. package/dist/generic/EntityDocumenTemplateParserBase.js.map +1 -1
  16. package/dist/generic/EntityDocumentTemplateParser.d.ts +16 -1
  17. package/dist/generic/EntityDocumentTemplateParser.d.ts.map +1 -1
  18. package/dist/generic/EntityDocumentTemplateParser.js +34 -16
  19. package/dist/generic/EntityDocumentTemplateParser.js.map +1 -1
  20. package/dist/generic/entitySyncConfig.types.d.ts +25 -0
  21. package/dist/generic/entitySyncConfig.types.d.ts.map +1 -1
  22. package/dist/generic/entitySyncConfig.types.js +1 -2
  23. package/dist/generic/vectorSync.types.d.ts +20 -0
  24. package/dist/generic/vectorSync.types.d.ts.map +1 -1
  25. package/dist/generic/vectorSync.types.js +1 -2
  26. package/dist/index.d.ts +4 -4
  27. package/dist/index.js +4 -20
  28. package/dist/index.js.map +1 -1
  29. package/dist/models/BatchWorker.d.ts +33 -2
  30. package/dist/models/BatchWorker.d.ts.map +1 -1
  31. package/dist/models/BatchWorker.js +30 -16
  32. package/dist/models/BatchWorker.js.map +1 -1
  33. package/dist/models/EntityDocumentCache.d.ts +3 -0
  34. package/dist/models/EntityDocumentCache.d.ts.map +1 -1
  35. package/dist/models/EntityDocumentCache.js +17 -17
  36. package/dist/models/EntityDocumentCache.js.map +1 -1
  37. package/dist/models/PagedRecords.d.ts +10 -1
  38. package/dist/models/PagedRecords.d.ts.map +1 -1
  39. package/dist/models/PagedRecords.js +13 -6
  40. package/dist/models/PagedRecords.js.map +1 -1
  41. package/dist/models/entityVectorSync.d.ts +32 -1
  42. package/dist/models/entityVectorSync.d.ts.map +1 -1
  43. package/dist/models/entityVectorSync.js +136 -73
  44. package/dist/models/entityVectorSync.js.map +1 -1
  45. package/dist/models/workers/UpsertVectors.d.ts.map +1 -1
  46. package/dist/models/workers/UpsertVectors.js +12 -17
  47. package/dist/models/workers/UpsertVectors.js.map +1 -1
  48. package/dist/models/workers/VectorizeTemplates.js +23 -20
  49. package/dist/models/workers/VectorizeTemplates.js.map +1 -1
  50. package/dist/models/workers/upserter.js +5 -8
  51. package/dist/models/workers/upserter.js.map +1 -1
  52. package/package.json +19 -18
package/README.md CHANGED
@@ -1,15 +1,54 @@
1
1
  # @memberjunction/ai-vector-sync
2
2
 
3
- A robust MemberJunction package for synchronizing entities with vector databases by transforming entity records into vector representations using embedding models.
3
+ Synchronizes MemberJunction entity records with vector databases by transforming records into embeddings through a template-based pipeline. Handles batch processing, worker-based parallelism, Entity Document management, and Entity Record Document tracking.
4
4
 
5
- ## Overview
5
+ ## Architecture
6
6
 
7
- The `@memberjunction/ai-vector-sync` package provides a comprehensive solution for:
8
- - Converting MemberJunction entities into vector embeddings
9
- - Storing embeddings in vector databases (currently supports Pinecone)
10
- - Managing the synchronization lifecycle between entities and their vector representations
11
- - Supporting batch processing for large datasets
12
- - Providing template-based document generation for vectorization
7
+ ```mermaid
8
+ graph TD
9
+ subgraph SyncPkg["@memberjunction/ai-vector-sync"]
10
+ EVS["EntityVectorSyncer"]
11
+ EDC["EntityDocumentCache"]
12
+ EDTP["EntityDocumentTemplateParser"]
13
+ BW["BatchWorker"]
14
+ end
15
+
16
+ subgraph Pipeline["Vectorization Pipeline"]
17
+ FETCH["Fetch Records<br/>(batched)"] --> TEMPL["Parse Templates<br/>(text from fields)"]
18
+ TEMPL --> EMBED["Generate Embeddings<br/>(AI model)"]
19
+ EMBED --> UPSERT["Upsert to<br/>Vector DB"]
20
+ UPSERT --> TRACK["Create Entity<br/>Record Documents"]
21
+ end
22
+
23
+ subgraph MJEntities["MemberJunction Entities"]
24
+ ED["Entity Documents"]
25
+ EDT["Entity Document Types"]
26
+ ERD["Entity Record Documents"]
27
+ VDI["Vector Indexes"]
28
+ end
29
+
30
+ subgraph External["External Services"]
31
+ AI["Embedding Model<br/>(OpenAI, Mistral, etc.)"]
32
+ VDB["Vector Database<br/>(Pinecone, etc.)"]
33
+ end
34
+
35
+ EVS --> EDC
36
+ EVS --> EDTP
37
+ EVS --> BW
38
+ EDTP --> TEMPL
39
+ BW --> EMBED
40
+ BW --> UPSERT
41
+ BW --> TRACK
42
+ EVS --> ED
43
+ EVS --> ERD
44
+ BW --> AI
45
+ BW --> VDB
46
+
47
+ style SyncPkg fill:#2d6a9f,stroke:#1a4971,color:#fff
48
+ style Pipeline fill:#2d8659,stroke:#1a5c3a,color:#fff
49
+ style MJEntities fill:#b8762f,stroke:#8a5722,color:#fff
50
+ style External fill:#7c5295,stroke:#563a6b,color:#fff
51
+ ```
13
52
 
14
53
  ## Installation
15
54
 
@@ -17,246 +56,284 @@ The `@memberjunction/ai-vector-sync` package provides a comprehensive solution f
17
56
  npm install @memberjunction/ai-vector-sync
18
57
  ```
19
58
 
20
- ## Prerequisites
59
+ ## Overview
21
60
 
22
- Before using this package, ensure you have:
61
+ This package converts MemberJunction entity records into vector embeddings stored in a vector database. The process is driven by **Entity Documents** -- metadata records that define which entity to vectorize, how to generate text from it (via templates), which embedding model to use, and where to store the results.
62
+
63
+ Key capabilities:
64
+
65
+ - **Batch processing** with configurable sizes for fetching, embedding, and upserting
66
+ - **Template-based text generation** using Entity Document templates that reference entity fields
67
+ - **Worker architecture** for concurrent embedding and upsert operations
68
+ - **Entity Document caching** via a singleton cache to avoid repeated database lookups
69
+ - **Default Entity Document creation** for entities that lack one
70
+ - **Resume support** via `StartingOffset` for interrupted processes
71
+ - **Entity Record Document tracking** to record which records have been vectorized
72
+
73
+ ## Vectorization Flow
74
+
75
+ ```mermaid
76
+ sequenceDiagram
77
+ participant Caller
78
+ participant EVS as EntityVectorSyncer
79
+ participant Cache as EntityDocumentCache
80
+ participant Parser as TemplateParser
81
+ participant Worker as BatchWorker
82
+ participant Model as Embedding Model
83
+ participant VDB as Vector Database
84
+ participant DB as MJ Database
85
+
86
+ Caller->>EVS: VectorizeEntity(params, user)
87
+ EVS->>EVS: Config(forceRefresh, user)
88
+ EVS->>Cache: Refresh (loads Entity Documents)
89
+ EVS->>Cache: GetDocument(entityDocumentID)
90
+ Cache-->>EVS: EntityDocumentEntity
91
+
92
+ EVS->>DB: Load template for Entity Document
93
+ EVS->>DB: Fetch entity records (batch)
94
+
95
+ loop For each batch
96
+ EVS->>Parser: Parse template for each record
97
+ Parser-->>EVS: Text strings
98
+
99
+ EVS->>Worker: VectorizeTemplates batch
100
+ Worker->>Model: createBatchEmbedding(texts)
101
+ Model-->>Worker: Embedding vectors
102
+
103
+ EVS->>Worker: UpsertVectors batch
104
+ Worker->>VDB: createRecords(vectors)
105
+ VDB-->>Worker: Success/failure
106
+
107
+ EVS->>Worker: Create EntityRecordDocuments
108
+ Worker->>DB: Save tracking records
109
+ end
110
+
111
+ EVS-->>Caller: VectorizeEntityResponse
112
+ ```
23
113
 
24
- 1. **SQL Database with MemberJunction Framework**
25
- A properly configured SQL database with the MemberJunction framework installed.
114
+ ## Core Components
26
115
 
27
- 2. **API Keys**
28
- - Embedding model API key (supports OpenAI, Mistral, etc.)
29
- - Vector database API key (currently supports Pinecone)
116
+ ### EntityVectorSyncer
30
117
 
31
- 3. **Entity Configuration**
32
- - Entity Document record defined in MemberJunction
33
- - Associated template for specifying which entity properties to vectorize
118
+ The main class that orchestrates the entire vectorization process. Extends `VectorBase` from `@memberjunction/ai-vectors`.
119
+
120
+ **Key methods:**
121
+
122
+ | Method | Description |
123
+ |---|---|
124
+ | `Config(forceRefresh, contextUser)` | Initializes engines and caches; must be called before vectorization |
125
+ | `VectorizeEntity(params, contextUser)` | Runs the full vectorization pipeline for an entity |
126
+ | `GetEntityDocument(id)` | Retrieves an Entity Document by ID |
127
+ | `GetEntityDocumentByName(name, user)` | Retrieves an Entity Document by name |
128
+ | `GetActiveEntityDocuments(entityNames?)` | Gets all active Entity Documents, optionally filtered |
129
+ | `CreateDefaultEntityDocument(entityID, vectorDB, aiModel)` | Creates a default Entity Document when one does not exist |
130
+
131
+ ### EntityDocumentCache
132
+
133
+ A singleton cache that loads all Entity Document and Entity Document Type records into memory for fast lookup.
134
+
135
+ ```mermaid
136
+ classDiagram
137
+ class EntityDocumentCache {
138
+ -_instance : EntityDocumentCache
139
+ -_cache : Record~string, EntityDocumentEntity~
140
+ -_typeCache : Record~string, EntityDocumentTypeEntity~
141
+ +Instance : EntityDocumentCache
142
+ +IsLoaded : boolean
143
+ +GetDocument(id) EntityDocumentEntity
144
+ +GetDocumentByName(name) EntityDocumentEntity
145
+ +GetDocumentType(id) EntityDocumentTypeEntity
146
+ +GetDocumentTypeByName(name) EntityDocumentTypeEntity
147
+ +GetFirstActiveDocumentForEntityByID(entityID) EntityDocumentEntity
148
+ +GetFirstActiveDocumentForEntityByName(name) EntityDocumentEntity
149
+ +Refresh(forceRefresh, user) void
150
+ +SetCurrentUser(user) void
151
+ }
152
+
153
+ style EntityDocumentCache fill:#2d6a9f,stroke:#1a4971,color:#fff
154
+ ```
34
155
 
35
- ## Core Features
156
+ ### EntityDocumentTemplateParser
36
157
 
37
- ### Entity Vectorization
38
- Transform entity records into high-dimensional vectors that capture the semantic meaning of the data.
158
+ Converts entity records into text strings by evaluating Entity Document templates. Templates use `${FieldName}` syntax to reference entity field values.
39
159
 
40
- ### Batch Processing
41
- Efficiently handle large datasets with configurable batch sizes for:
42
- - Record fetching
43
- - Vectorization
44
- - Database upsertion
160
+ ```typescript
161
+ // Template example: "${FirstName} ${LastName} works at ${Company} as ${Title}"
162
+ // With record { FirstName: 'Jane', LastName: 'Doe', Company: 'Acme', Title: 'Engineer' }
163
+ // Result: "Jane Doe works at Acme as Engineer"
164
+ ```
45
165
 
46
- ### Template-Based Processing
47
- Use MemberJunction templates to define which entity fields and relationships to include in vectorization.
166
+ ### BatchWorker
48
167
 
49
- ### Vector Database Integration
50
- Seamlessly integrate with vector databases through the MemberJunction AI infrastructure.
168
+ Handles the parallel execution of embedding generation, vector database upserts, and Entity Record Document creation. Configurable batch sizes allow tuning for memory and API rate limits.
51
169
 
52
170
  ## Usage
53
171
 
54
- ### Basic Entity Vectorization
172
+ ### Basic Vectorization
55
173
 
56
174
  ```typescript
57
175
  import { EntityVectorSyncer } from '@memberjunction/ai-vector-sync';
58
176
  import { UserInfo } from '@memberjunction/core';
59
177
 
60
- // Initialize the syncer
61
178
  const syncer = new EntityVectorSyncer();
62
179
 
63
- // Configure the syncer (required before first use)
180
+ // Initialize (required once)
64
181
  await syncer.Config(false, contextUser);
65
182
 
66
- // Vectorize an entity
67
- const params = {
68
- entityID: 'your-entity-id',
69
- entityDocumentID: 'your-entity-document-id',
70
- listBatchCount: 50, // Optional: records per batch (default: 50)
71
- VectorizeBatchCount: 50, // Optional: vectorization batch size (default: 50)
72
- UpsertBatchCount: 50, // Optional: upsert batch size (default: 50)
73
- StartingOffset: 0 // Optional: skip records for resuming
74
- };
75
-
76
- // Start vectorization (runs asynchronously)
77
- syncer.VectorizeEntity(params, contextUser);
183
+ // Vectorize all records for an entity
184
+ await syncer.VectorizeEntity({
185
+ entityID: 'entity-uuid',
186
+ entityDocumentID: 'doc-uuid',
187
+ listBatchCount: 50,
188
+ VectorizeBatchCount: 50,
189
+ UpsertBatchCount: 50
190
+ }, contextUser);
78
191
  ```
79
192
 
80
- ### Vectorizing a Specific List
193
+ ### Vectorize a Specific List
81
194
 
82
195
  ```typescript
83
- // Vectorize only records within a specific list
84
- const params = {
85
- entityID: 'your-entity-id',
86
- entityDocumentID: 'your-entity-document-id',
87
- listID: 'your-list-id', // Only vectorize records in this list
88
- listBatchCount: 100
89
- };
90
-
91
- await syncer.VectorizeEntity(params, contextUser);
196
+ await syncer.VectorizeEntity({
197
+ entityID: 'entity-uuid',
198
+ entityDocumentID: 'doc-uuid',
199
+ listID: 'list-uuid' // Only records in this list
200
+ }, contextUser);
92
201
  ```
93
202
 
94
- ### Working with Entity Documents
203
+ ### Resume Interrupted Processing
95
204
 
96
205
  ```typescript
97
- // Get entity document by ID
98
- const entityDoc = await syncer.GetEntityDocument('document-id');
99
-
100
- // Get entity document by name
101
- const entityDoc = await syncer.GetEntityDocumentByName('Document Name', contextUser);
102
-
103
- // Get all active entity documents
104
- const activeDocs = await syncer.GetActiveEntityDocuments();
105
-
106
- // Get active documents for specific entities
107
- const specificDocs = await syncer.GetActiveEntityDocuments(['Entity1', 'Entity2']);
206
+ await syncer.VectorizeEntity({
207
+ entityID: 'entity-uuid',
208
+ entityDocumentID: 'doc-uuid',
209
+ StartingOffset: 5000 // Skip first 5000 records
210
+ }, contextUser);
108
211
  ```
109
212
 
110
- ### Creating Default Entity Documents
213
+ ### Manage Entity Documents
111
214
 
112
215
  ```typescript
113
- import { VectorDatabaseEntity, AIModelEntity } from '@memberjunction/core-entities';
114
-
115
- // Create a default entity document when one doesn't exist
116
- const entityDoc = await syncer.CreateDefaultEntityDocument(
117
- entityID,
118
- vectorDatabase, // VectorDatabaseEntity instance
119
- aiModel // AIModelEntity instance
120
- );
121
- ```
122
-
123
- ## API Reference
124
-
125
- ### EntityVectorSyncer
126
-
127
- The main class for entity vectorization operations.
128
-
129
- #### Methods
216
+ // Look up by name
217
+ const doc = await syncer.GetEntityDocumentByName('Contacts Vectorization', contextUser);
130
218
 
131
- ##### `Config(forceRefresh: boolean, contextUser?: UserInfo): Promise<void>`
132
- Configures the syncer and initializes required engines.
133
- - `forceRefresh`: Force refresh of caches and engines
134
- - `contextUser`: User context for operations
135
-
136
- ##### `VectorizeEntity(params: VectorizeEntityParams, contextUser?: UserInfo): Promise<VectorizeEntityResponse>`
137
- Vectorizes entities based on provided parameters.
138
- - `params`: Configuration for vectorization
139
- - `contextUser`: Required user context
140
-
141
- ##### `GetEntityDocument(entityDocumentID: string): Promise<EntityDocumentEntity | null>`
142
- Retrieves an entity document by ID.
219
+ // Get all active documents
220
+ const activeDocs = await syncer.GetActiveEntityDocuments();
143
221
 
144
- ##### `GetEntityDocumentByName(entityDocumentName: string, contextUser?: UserInfo): Promise<EntityDocumentEntity | null>`
145
- Retrieves an entity document by name.
222
+ // Get active documents for specific entities only
223
+ const filtered = await syncer.GetActiveEntityDocuments(['Contacts', 'Companies']);
146
224
 
147
- ##### `GetActiveEntityDocuments(entityNames?: string[]): Promise<EntityDocumentEntity[]>`
148
- Gets all active entity documents, optionally filtered by entity names.
225
+ // Create a default document when none exists
226
+ const newDoc = await syncer.CreateDefaultEntityDocument(
227
+ entityID, vectorDatabase, aiModel
228
+ );
229
+ ```
149
230
 
150
- ##### `CreateDefaultEntityDocument(entityID: string, vectorDatabase: VectorDatabaseEntity, aiModel: AIModelEntity): Promise<EntityDocumentEntity>`
151
- Creates a default entity document for the specified entity.
231
+ ## Configuration Types
152
232
 
153
- ### Types
233
+ ### VectorizeEntityParams
154
234
 
155
- #### VectorizeEntityParams
156
235
  ```typescript
157
236
  type VectorizeEntityParams = {
158
- entityID: string; // Required: Entity to vectorize
159
- entityDocumentID?: string; // Entity document configuration
160
- listID?: string; // Optional: Specific list to vectorize
161
- listBatchCount?: number; // Records per fetch batch (default: 50)
162
- VectorizeBatchCount?: number; // Vectorization batch size (default: 50)
163
- UpsertBatchCount?: number; // Database upsert batch size (default: 50)
164
- StartingOffset?: number; // Skip records for resuming
165
- CurrentUser?: UserInfo; // User context
166
- options?: any; // Additional options
167
- }
237
+ entityID: string; // Entity to vectorize
238
+ entityDocumentID?: string; // Entity Document configuration
239
+ listID?: string; // Optional: vectorize only this list
240
+ listBatchCount?: number; // Records per fetch batch (default: 50)
241
+ VectorizeBatchCount?: number; // Embedding batch size (default: 50)
242
+ UpsertBatchCount?: number; // DB upsert batch size (default: 50)
243
+ StartingOffset?: number; // Skip records for resume
244
+ CurrentUser?: UserInfo; // User context
245
+ };
168
246
  ```
169
247
 
170
- #### EntitySyncConfig
248
+ ### EntitySyncConfig
249
+
171
250
  ```typescript
172
251
  type EntitySyncConfig = {
173
- EntityDocumentID: string; // Entity document to use
174
- Interval: number; // Sync interval in seconds
175
- RunViewParams: RunViewParams; // View parameters for fetching records
176
- IncludeInSync: boolean; // Include in sync process
177
- LastRunDate: string; // Last sync timestamp
178
- VectorIndexID: number; // Vector index ID
179
- VectorID: number; // Vector database ID
180
- }
252
+ EntityDocumentID: string;
253
+ Interval: number; // Seconds between syncs
254
+ RunViewParams: RunViewParams;
255
+ IncludeInSync: boolean;
256
+ LastRunDate: string;
257
+ VectorIndexID: number;
258
+ VectorID: number;
259
+ };
181
260
  ```
182
261
 
183
- ## Architecture
184
-
185
- ### Process Flow
186
-
187
- 1. **Entity Document Retrieval**: Fetches configuration from Entity Document record
188
- 2. **Model and Database Configuration**: Sets up embedding model and vector database
189
- 3. **Data Fetching**: Retrieves entity records in batches
190
- 4. **Vectorization**: Transforms records using embedding model
191
- 5. **Vector Upsertion**: Stores vectors in database
192
- 6. **EntityRecordDocument Creation**: Creates tracking records
193
-
194
- ### Worker Architecture
195
-
196
- The package uses a multi-worker architecture for efficient processing:
197
- - **VectorizeTemplates Worker**: Handles template-based text generation and embedding
198
- - **UpsertVectors Worker**: Manages vector database operations
199
- - **EntityRecordDocument Worker**: Tracks vector-entity relationships
200
-
201
- ## Configuration
202
-
203
- ### Environment Variables
262
+ ## Entity Document Templates
263
+
264
+ Templates define how entity records are transformed into text for embedding generation.
265
+
266
+ ```mermaid
267
+ graph LR
268
+ ED["Entity Document"] --> TMPL["Template<br/>${Field} syntax"]
269
+ TMPL --> PARSER["Template Parser"]
270
+ REC["Entity Record"] --> PARSER
271
+ PARSER --> TEXT["Plain Text"]
272
+ TEXT --> EMBED["Embedding Model"]
273
+ EMBED --> VEC["Vector"]
274
+
275
+ style ED fill:#2d6a9f,stroke:#1a4971,color:#fff
276
+ style TMPL fill:#2d8659,stroke:#1a5c3a,color:#fff
277
+ style PARSER fill:#b8762f,stroke:#8a5722,color:#fff
278
+ style EMBED fill:#7c5295,stroke:#563a6b,color:#fff
279
+ style REC fill:#2d8659,stroke:#1a5c3a,color:#fff
280
+ style TEXT fill:#b8762f,stroke:#8a5722,color:#fff
281
+ style VEC fill:#7c5295,stroke:#563a6b,color:#fff
282
+ ```
204
283
 
205
- Create a `.env` file with:
284
+ ## Environment Variables
206
285
 
207
286
  ```env
208
- # Database Configuration
209
- DB_HOST=your-database-host
287
+ # Database
288
+ DB_HOST=your-sql-server
210
289
  DB_PORT=1433
211
290
  DB_USERNAME=your-username
212
291
  DB_PASSWORD=your-password
213
292
  DB_DATABASE=your-database
214
293
 
215
- # API Keys
294
+ # AI Models
216
295
  OPENAI_API_KEY=your-openai-key
217
296
  MISTRAL_API_KEY=your-mistral-key
297
+
298
+ # Vector Database
218
299
  PINECONE_API_KEY=your-pinecone-key
219
300
  PINECONE_HOST=your-pinecone-host
220
301
  PINECONE_DEFAULT_INDEX=your-default-index
221
302
 
222
- # User Configuration
303
+ # User Context
223
304
  CURRENT_USER_EMAIL=user@example.com
224
305
  ```
225
306
 
226
- ## Performance Considerations
227
-
228
- - **Long-Running Processes**: Vectorization can take hours for large datasets
229
- - **Batch Sizes**: Adjust batch sizes based on your system resources
230
- - **Asynchronous Processing**: Consider running vectorization in background processes
231
- - **Memory Usage**: Monitor memory usage for large batch sizes
307
+ ## Dependencies
232
308
 
233
- ## Integration with MemberJunction
309
+ | Package | Purpose |
310
+ |---|---|
311
+ | `@memberjunction/ai` | `BaseEmbeddings`, `GetAIAPIKey`, `EmbedTextsResult` |
312
+ | `@memberjunction/ai-vectordb` | `VectorDBBase`, `VectorRecord` |
313
+ | `@memberjunction/ai-vectors` | `VectorBase` base class |
314
+ | `@memberjunction/aiengine` | `AIEngine` singleton |
315
+ | `@memberjunction/core` | `Metadata`, `RunView`, `BaseEntity`, `UserInfo` |
316
+ | `@memberjunction/core-entities` | Entity type definitions |
317
+ | `@memberjunction/global` | MJGlobal class factory |
318
+ | `@memberjunction/templates` | Template engine for text generation |
234
319
 
235
- This package integrates seamlessly with:
236
- - `@memberjunction/core`: Core entity and metadata functionality
237
- - `@memberjunction/ai`: AI model abstractions
238
- - `@memberjunction/ai-vectordb`: Vector database abstractions
239
- - `@memberjunction/templates`: Template processing engine
320
+ ## Performance Considerations
240
321
 
241
- ## Error Handling
322
+ - **Batch sizes**: Adjust `listBatchCount`, `VectorizeBatchCount`, and `UpsertBatchCount` based on available memory and API rate limits
323
+ - **Long-running**: Full vectorization of large entities can take hours; use `StartingOffset` to resume
324
+ - **Worker concurrency**: The BatchWorker processes embedding and upsert operations concurrently within each batch
325
+ - **Caching**: `EntityDocumentCache` reduces database lookups for document metadata
242
326
 
243
- The package includes comprehensive error handling:
244
- - Validation of entity documents and templates
245
- - Graceful handling of API failures
246
- - Detailed logging through MemberJunction's logging system
327
+ ## Development
247
328
 
248
- ## Best Practices
329
+ ```bash
330
+ # Build
331
+ npm run build
249
332
 
250
- 1. **Start with Small Batches**: Test with small batch sizes before processing large datasets
251
- 2. **Monitor Progress**: Use MemberJunction's logging to track vectorization progress
252
- 3. **Handle Interruptions**: Use `StartingOffset` to resume interrupted processes
253
- 4. **Template Design**: Design templates to include relevant fields for semantic search
254
- 5. **Resource Management**: Consider database and API rate limits when setting batch sizes
333
+ # Development mode
334
+ npm run start
335
+ ```
255
336
 
256
337
  ## License
257
338
 
258
- ISC - See LICENSE file for details
259
-
260
- ## Author
261
-
262
- MemberJunction.com
339
+ ISC
package/dist/config.js CHANGED
@@ -1,21 +1,15 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.mistralAPIKey = exports.currentUserEmail = exports.serverPort = exports.dbDatabase = exports.dbPassword = exports.dbUsername = exports.dbPort = exports.dbHost = exports.pineconeDefaultIndex = exports.pineconeAPIKey = exports.pineconeHost = exports.openAIAPIKey = void 0;
7
- const dotenv_1 = __importDefault(require("dotenv"));
8
- dotenv_1.default.config();
9
- exports.openAIAPIKey = process.env.OPENAI_API_KEY;
10
- exports.pineconeHost = process.env.PINECONE_HOST;
11
- exports.pineconeAPIKey = process.env.PINECONE_API_KEY;
12
- exports.pineconeDefaultIndex = process.env.PINECONE_DEFAULT_INDEX;
13
- exports.dbHost = process.env.DB_HOST;
14
- exports.dbPort = Number(process.env.DB_PORT) || 1433;
15
- exports.dbUsername = process.env.DB_USERNAME;
16
- exports.dbPassword = process.env.DB_PASSWORD;
17
- exports.dbDatabase = process.env.DB_DATABASE;
18
- exports.serverPort = Number(process.env.PORT) || 8000;
19
- exports.currentUserEmail = process.env.CURRENT_USER_EMAIL;
20
- exports.mistralAPIKey = process.env.MISTRAL_API_KEY;
1
+ import dotenv from 'dotenv';
2
+ dotenv.config({ quiet: true });
3
+ export const openAIAPIKey = process.env.OPENAI_API_KEY;
4
+ export const pineconeHost = process.env.PINECONE_HOST;
5
+ export const pineconeAPIKey = process.env.PINECONE_API_KEY;
6
+ export const pineconeDefaultIndex = process.env.PINECONE_DEFAULT_INDEX;
7
+ export const dbHost = process.env.DB_HOST;
8
+ export const dbPort = Number(process.env.DB_PORT) || 1433;
9
+ export const dbUsername = process.env.DB_USERNAME;
10
+ export const dbPassword = process.env.DB_PASSWORD;
11
+ export const dbDatabase = process.env.DB_DATABASE;
12
+ export const serverPort = Number(process.env.PORT) || 8000;
13
+ export const currentUserEmail = process.env.CURRENT_USER_EMAIL;
14
+ export const mistralAPIKey = process.env.MISTRAL_API_KEY;
21
15
  //# sourceMappingURL=config.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAC;AAEH,QAAA,YAAY,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;AAClD,QAAA,YAAY,GAAW,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;AACjD,QAAA,cAAc,GAAW,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;AACtD,QAAA,oBAAoB,GAAW,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;AAElE,QAAA,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;AAC7B,QAAA,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;AAC7C,QAAA,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AACrC,QAAA,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AACrC,QAAA,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AACrC,QAAA,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AAE9C,QAAA,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;AAElD,QAAA,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC"}
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE/B,MAAM,CAAC,MAAM,YAAY,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;AAC/D,MAAM,CAAC,MAAM,YAAY,GAAW,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;AAC9D,MAAM,CAAC,MAAM,cAAc,GAAW,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;AACnE,MAAM,CAAC,MAAM,oBAAoB,GAAW,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;AAE/E,MAAM,CAAC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;AAC1C,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;AAC1D,MAAM,CAAC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AAClD,MAAM,CAAC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AAClD,MAAM,CAAC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AAClD,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AAE3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;AAE/D,MAAM,CAAC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC"}
package/dist/db/db.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import * as mssql from 'mssql';
1
+ import mssql from 'mssql';
2
2
  declare const SQLConnectionPool: mssql.ConnectionPool;
3
3
  export default SQLConnectionPool;
4
4
  //# sourceMappingURL=db.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAc/B,QAAA,MAAM,iBAAiB,sBAAmC,CAAC;AAE3D,eAAe,iBAAiB,CAAC"}
1
+ {"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/db/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAc1B,QAAA,MAAM,iBAAiB,sBAAmC,CAAC;AAE3D,eAAe,iBAAiB,CAAC"}
package/dist/db/db.js CHANGED
@@ -1,40 +1,15 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- const mssql = __importStar(require("mssql"));
27
- const config_1 = require("../config");
1
+ import mssql from 'mssql';
2
+ import { dbDatabase, dbHost, dbPassword, dbPort, dbUsername } from '../config.js';
28
3
  const config = {
29
- user: config_1.dbUsername,
30
- password: config_1.dbPassword,
31
- server: config_1.dbHost,
32
- port: config_1.dbPort,
33
- database: config_1.dbDatabase,
4
+ user: dbUsername,
5
+ password: dbPassword,
6
+ server: dbHost,
7
+ port: dbPort,
8
+ database: dbDatabase,
34
9
  options: {
35
10
  encrypt: true,
36
11
  },
37
12
  };
38
13
  const SQLConnectionPool = new mssql.ConnectionPool(config);
39
- exports.default = SQLConnectionPool;
14
+ export default SQLConnectionPool;
40
15
  //# sourceMappingURL=db.js.map
package/dist/db/db.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/db/db.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA+B;AAC/B,sCAA+E;AAE/E,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,mBAAU;IAChB,QAAQ,EAAE,mBAAU;IACpB,MAAM,EAAE,eAAM;IACd,IAAI,EAAE,eAAM;IACZ,QAAQ,EAAE,mBAAU;IACpB,OAAO,EAAE;QACP,OAAO,EAAE,IAAI;KACd;CACF,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;AAE3D,kBAAe,iBAAiB,CAAC"}
1
+ {"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/db/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE/E,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,UAAU;IAChB,QAAQ,EAAE,UAAU;IACpB,MAAM,EAAE,MAAM;IACd,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE;QACP,OAAO,EAAE,IAAI;KACd;CACF,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;AAE3D,eAAe,iBAAiB,CAAC"}
package/dist/db/dbAI.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import * as sql from 'mssql';
1
+ import sql from 'mssql';
2
2
  declare const pool: sql.ConnectionPool;
3
3
  export default pool;
4
4
  //# sourceMappingURL=dbAI.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"dbAI.d.ts","sourceRoot":"","sources":["../../src/db/dbAI.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,GAAG,MAAM,OAAO,CAAC;AAgB7B,QAAA,MAAM,IAAI,oBAAiC,CAAC;AAE5C,eAAe,IAAI,CAAC"}
1
+ {"version":3,"file":"dbAI.d.ts","sourceRoot":"","sources":["../../src/db/dbAI.ts"],"names":[],"mappings":"AAEA,OAAO,GAAG,MAAM,OAAO,CAAC;AAgBxB,QAAA,MAAM,IAAI,oBAAiC,CAAC;AAE5C,eAAe,IAAI,CAAC"}