@memberjunction/ai-vector-sync 5.21.0 → 5.22.0

package/dist/generic/entityDocumentConfig.types.d.ts

@@ -0,0 +1,185 @@
+/**
+ * Configuration interface for EntityDocument.Configuration JSON column.
+ *
+ * Controls how the vectorization pipeline processes records for a given
+ * entity document — which fields get into vector metadata, how they're
+ * truncated, how records are filtered, and pipeline tuning knobs.
+ *
+ * All fields are optional. NULL / missing means "use system defaults."
+ */
+export interface EntityDocumentConfiguration {
+    /** Controls which entity fields appear in vector metadata and how they're stored */
+    metadata?: EntityDocumentMetadataConfig;
+    /** Scheduling settings for automated incremental sync */
+    scheduling?: EntityDocumentSchedulingConfig;
+    /** Pipeline tuning — batch sizes, concurrency, rate limiting */
+    pipeline?: EntityDocumentPipelineConfig;
+    /** Record filtering — limit which records get vectorized */
+    recordFilter?: EntityDocumentRecordFilterConfig;
+    /** Content chunking for long-form text fields */
+    chunking?: EntityDocumentChunkingConfig;
+}
+export interface EntityDocumentMetadataConfig {
+    /**
+     * Strategy for selecting which fields go into vector metadata:
+     * - "all":     include every non-PK, non-FK, non-binary, non-system field
+     *              (current default behavior)
+     * - "include": only include fields listed in `fields`
+     * - "exclude": include all fields EXCEPT those listed in `fields`
+     */
+    fieldStrategy?: 'all' | 'include' | 'exclude';
+    /**
+     * Per-field overrides. The key is the entity field name.
+     * Only relevant when fieldStrategy is "include" or "exclude",
+     * or when you need to override truncation for specific fields
+     * under "all" mode.
+     */
+    fields?: Record<string, EntityDocumentFieldConfig>;
+    /**
+     * Global default truncation limit in characters for fields with
+     * MaxLength > 5000 or nvarchar(MAX). Default: 1000.
+     * Individual field overrides in `fields` take precedence.
+     */
+    defaultTruncationLimit?: number;
+    /**
+     * Whether to include the entity icon in vector metadata.
+     * Default: true.
+     */
+    includeEntityIcon?: boolean;
+    /**
+     * Whether to include __mj_UpdatedAt in vector metadata for
+     * freshness display in search results. Default: true.
+     */
+    includeUpdatedAt?: boolean;
+}
+export interface EntityDocumentFieldConfig {
+    /**
+     * Whether this field is included in vector metadata.
+     * Used with "include" strategy to explicitly list fields,
+     * or with "exclude" strategy to mark specific fields for exclusion.
+     * Under "all" strategy, set to false to exclude a specific field.
+     */
+    included?: boolean;
+    /**
+     * Override the truncation limit for this specific field (in characters).
+     * Only applies to large text fields. NULL means use the global
+     * defaultTruncationLimit.
+     */
+    truncationLimit?: number;
+    /**
+     * Whether this field should be used as the primary display title
+     * in search result cards. At most one field should be marked.
+     * If none are marked, the system uses IsNameField from entity metadata.
+     */
+    isDisplayTitle?: boolean;
+    /**
+     * Whether this field should be used as the snippet/preview text
+     * in search result cards. At most one field should be marked.
+     */
+    isSnippetField?: boolean;
+}
+export interface EntityDocumentSchedulingConfig {
+    /**
+     * Whether automated sync is enabled for this entity document.
+     * Default: false (manual sync only).
+     */
+    enabled?: boolean;
+    /**
+     * Cron expression for scheduled sync (e.g. "0 2 * * *" for daily at 2 AM).
+     * Mutually exclusive with `intervalSeconds`.
+     */
+    cronExpression?: string;
+    /**
+     * Interval in seconds between sync runs. Only used if cronExpression
+     * is not set. Minimum: 300 (5 minutes).
+     */
+    intervalSeconds?: number;
+    /**
+     * Whether to run a full sync or only incremental (records changed
+     * since last sync). Default: "incremental".
+     */
+    syncMode?: 'full' | 'incremental';
+    /**
+     * For incremental mode, the timestamp column used to detect changes.
+     * Default: "__mj_UpdatedAt".
+     */
+    changeTrackingColumn?: string;
+}
+export interface EntityDocumentPipelineConfig {
+    /**
+     * Number of records to fetch from the database per page.
+     * Default: 100.
+     */
+    fetchBatchSize?: number;
+    /**
+     * Number of records to render templates and embed in parallel.
+     * Default: 50.
+     */
+    vectorizeBatchSize?: number;
+    /**
+     * Number of vectors to upsert to the vector DB in a single call.
+     * Default: 50. Capped by VectorDatabaseConfiguration.throughput.maxUpsertBatchSize.
+     */
+    upsertBatchSize?: number;
+    /**
+     * Maximum number of concurrent embedding API calls.
+     * Controls parallelism in the AsyncBatchTransform pipeline.
+     * Default: 3.
+     */
+    maxConcurrentEmbeddings?: number;
+    /**
+     * Delay in milliseconds between API calls to avoid rate limiting.
+     * Default: 0 (no delay).
+     */
+    delayBetweenCallsMs?: number;
+    /**
+     * Whether to force re-vectorization of records that have already
+     * been vectorized (by their deterministic SHA-1 ID). Default: false.
+     */
+    forceRevectorize?: boolean;
+}
+export interface EntityDocumentRecordFilterConfig {
+    /**
+     * Additional SQL filter predicate appended to the WHERE clause when
+     * fetching records to vectorize. Uses the same syntax as RunView's
+     * ExtraFilter (e.g. "Status = 'Active' AND IsDeleted = 0").
+     */
+    extraFilter?: string;
+    /**
+     * Maximum number of records to vectorize in a single run.
+     * 0 or undefined means no limit.
+     */
+    maxRecords?: number;
+    /**
+     * Starting offset — skip this many records before vectorizing.
+     * Useful for resuming after a partial failure.
+     */
+    startingOffset?: number;
+}
+export interface EntityDocumentChunkingConfig {
+    /**
+     * Whether to enable chunking for long-form text content.
+     * When enabled, records with text exceeding `maxChunkTokens` are
+     * split into multiple vectors with shared metadata.
+     * Default: false (entire rendered template = one vector).
+     */
+    enabled?: boolean;
+    /**
+     * Maximum tokens per chunk. The embedding model's context window
+     * is the hard ceiling. Default: 512.
+     */
+    maxChunkTokens?: number;
+    /**
+     * Number of tokens to overlap between consecutive chunks.
+     * Ensures context continuity at chunk boundaries. Default: 50.
+     */
+    overlapTokens?: number;
+    /**
+     * Chunking strategy:
+     * - "token":     split on token boundaries (most precise)
+     * - "sentence":  split on sentence boundaries (more readable)
+     * - "paragraph": split on paragraph/double-newline boundaries
+     */
+    strategy?: 'token' | 'sentence' | 'paragraph';
+}
+//# sourceMappingURL=entityDocumentConfig.types.d.ts.map

package/dist/generic/entityDocumentConfig.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entityDocumentConfig.types.d.ts","sourceRoot":"","sources":["../../src/generic/entityDocumentConfig.types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,MAAM,WAAW,2BAA2B;IACxC,oFAAoF;IACpF,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IAExC,yDAAyD;IACzD,UAAU,CAAC,EAAE,8BAA8B,CAAC;IAE5C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IAExC,4DAA4D;IAC5D,YAAY,CAAC,EAAE,gCAAgC,CAAC;IAEhD,iDAAiD;IACjD,QAAQ,CAAC,EAAE,4BAA4B,CAAC;CAC3C;AAMD,MAAM,WAAW,4BAA4B;IACzC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,SAAS,CAAC;IAE9C;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;IAEnD;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAMD,MAAM,WAAW,8BAA8B;IAC3C;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IAElC;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACjC;AAMD,MAAM,WAAW,4BAA4B;IACzC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAMD,MAAM,WAAW,gCAAgC;IAC7C;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAMD,MAAM,WAAW,4BAA4B;IACzC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,WAAW,CAAC;CACjD"}

package/dist/generic/entityDocumentConfig.types.js

@@ -0,0 +1,11 @@
+/**
+ * Configuration interface for EntityDocument.Configuration JSON column.
+ *
+ * Controls how the vectorization pipeline processes records for a given
+ * entity document — which fields get into vector metadata, how they're
+ * truncated, how records are filtered, and pipeline tuning knobs.
+ *
+ * All fields are optional. NULL / missing means "use system defaults."
+ */
+export {};
+//# sourceMappingURL=entityDocumentConfig.types.js.map

package/dist/generic/entityDocumentConfig.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"entityDocumentConfig.types.js","sourceRoot":"","sources":["../../src/generic/entityDocumentConfig.types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}

package/dist/generic/vectorSync.types.d.ts

@@ -31,7 +31,34 @@ export type VectorizeEntityParams = {
      * The number of records to skip before starting to fetch records from the list.
      */
     StartingOffset?: number;
-    options?: any;
+    options?: VectorizeEntityOptions;
+    /**
+     * Optional callback invoked during vectorization with progress updates.
+     * Called after each batch is upserted to the vector database.
+     */
+    OnProgress?: (update: VectorizeProgressUpdate) => void;
+};
+export type VectorizeEntityOptions = {
+    /** Delay in ms between API calls to avoid rate limiting */
+    delayTimeMS?: number;
+    /** Whether to force re-vectorization of already-vectorized records */
+    forceRevectorize?: boolean;
+};
+/**
+ * Progress update emitted during vectorization.
+ * Pass a callback via VectorizeEntityParams.OnProgress to receive updates.
+ */
+export type VectorizeProgressUpdate = {
+    /** Total records to process (may increase as pages are fetched) */
+    TotalRecords: number;
+    /** Number of records processed so far */
+    ProcessedRecords: number;
+    /** Current pipeline stage */
+    Stage: 'embedding' | 'upserting' | 'complete' | 'error';
+    /** Percent complete (0-100) */
+    PercentComplete: number;
+    /** Elapsed time in ms since pipeline start */
+    ElapsedMs: number;
 };
 export type VectorizeEntityResponse = {
     success: boolean;
@@ -41,13 +68,14 @@ export type VectorizeEntityResponse = {
 export type EmbeddingData = {
     ID: number;
     Vector: number[];
-    VectorID?: unknown;
-    EntityData: Record<string, any>;
-    __mj_recordID: unknown;
-    __mj_compositeKey?: unknown;
+    VectorID?: string;
+    EntityData: Record<string, unknown>;
+    __mj_recordID: string | number;
+    __mj_compositeKey?: string;
+    /** Plain object representation of the entity document */
     EntityDocument?: Record<string, unknown>;
     TemplateContent?: string;
-    VectorIndexID?: unknown;
+    VectorIndexID?: string;
 };
 export type VectorEmeddingData = {
     embedding: BaseEmbeddings;
@@ -57,6 +85,10 @@ export type VectorEmeddingData = {
     embeddingDriverClass: string;
     embeddingAPIKey: string;
 };
+/**
+ * @deprecated Worker contexts are no longer used since worker_threads were replaced
+ * with main-thread async processing. Kept for backward compatibility.
+ */
 export type AnnotateWorkerContext = {
     executionId: number;
     entity: BaseEntity;
@@ -67,6 +99,10 @@ export type AnnotateWorkerContext = {
     embeddingAPIKey: string;
     delayTimeMS: number;
 };
+/**
+ * @deprecated Worker contexts are no longer used since worker_threads were replaced
+ * with main-thread async processing. Kept for backward compatibility.
+ */
 export type ArchiveWorkerContext = {
     executionId: number;
     entity: BaseEntity;

package/dist/generic/vectorSync.types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vectorSync.types.d.ts","sourceRoot":"","sources":["../../src/generic/vectorSync.types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAE1H,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;MAIE;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,GAAG,CAAC;CACjB,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACxB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC7B,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,YAAY,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAA;CAC1B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,eAAe,EAAE,uBAAuB,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,cAAc,EAAE,sBAAsB,CAAC;IACvC,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,uBAAuB,CAAC;IACzC,UAAU,EAAE,gBAAgB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,OAAO,EAAE,CAAC;CACjB,CAAC"}
+{"version":3,"file":"vectorSync.types.d.ts","sourceRoot":"","sources":["../../src/generic/vectorSync.types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAE1H,MAAM,MAAM,qBAAqB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;MAIE;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,sBAAsB,CAAC;IACjC;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,uBAAuB,KAAK,IAAI,CAAC;CAC1D,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACjC,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B,CAAA;AAED;;;GAGG;AACH,MAAM,MAAM,uBAAuB,GAAG;IAClC,mEAAmE;IACnE,YAAY,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,gBAAgB,EAAE,MAAM,CAAC;IACzB,6BAA6B;IAC7B,KAAK,EAAE,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,OAAO,CAAC;IACxD,+BAA+B;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACrB,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACxB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC7B,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,YAAY,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAA;CAC1B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,eAAe,EAAE,uBAAuB,CAAC;IACzC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEJ;;;GAGG;AACH,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,cAAc,EAAE,sBAAsB,CAAC;IACvC,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,uBAAuB,CAAC;IACzC,UAAU,EAAE,gBAAgB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,OAAO,EAAE,CAAC;CACjB,CAAC"}

package/dist/index.d.ts

@@ -1,5 +1,7 @@
 export * from './generic/vectorSync.types.js';
 export * from './generic/entitySyncConfig.types.js';
+export * from './generic/entityDocumentConfig.types.js';
 export * from './models/entityVectorSync.js';
+export * from './models/EntityDocumentSuggester.js';
 export * from './generic/EntityDocumentTemplateParser.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAC;AAC3C,cAAc,kCAAkC,CAAC;AACjD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,wCAAwC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAC;AAC3C,cAAc,kCAAkC,CAAC;AACjD,cAAc,sCAAsC,CAAC;AACrD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,kCAAkC,CAAC;AACjD,cAAc,wCAAwC,CAAC"}

package/dist/index.js

@@ -1,5 +1,7 @@
 export * from './generic/vectorSync.types.js';
 export * from './generic/entitySyncConfig.types.js';
+export * from './generic/entityDocumentConfig.types.js';
 export * from './models/entityVectorSync.js';
+export * from './models/EntityDocumentSuggester.js';
 export * from './generic/EntityDocumentTemplateParser.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAC;AAC3C,cAAc,kCAAkC,CAAC;AACjD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,wCAAwC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAC;AAC3C,cAAc,kCAAkC,CAAC;AACjD,cAAc,sCAAsC,CAAC;AACrD,cAAc,2BAA2B,CAAC;AAC1C,cAAc,kCAAkC,CAAC;AACjD,cAAc,wCAAwC,CAAC"}

package/dist/models/AsyncBatchTransform.d.ts

@@ -0,0 +1,44 @@
+import { Transform } from 'node:stream';
+export type TransformCallback = Parameters<Transform['_flush']>[0];
+/**
+ * Options for the AsyncBatchTransform
+ */
+export type AsyncBatchTransformOptions<TRecord, TContext, TResult> = {
+    /** Number of records to accumulate before processing a batch. Default: 10 */
+    batchSize?: number;
+    /** Max concurrent batch processing tasks. Default: 4 */
+    concurrencyLimit?: number;
+    /** Arbitrary context passed to the processing function */
+    context?: TContext;
+    /** The async function that processes a batch of records and returns results */
+    processBatch: (batch: TRecord[], context: TContext) => Promise<TResult[]>;
+};
+/**
+ * A Transform stream that batches records and processes them asynchronously
+ * in the main thread using a concurrency-limited queue.
+ *
+ * This replaces the worker_threads-based BatchWorker to solve the ClassFactory
+ * registration issue: worker threads run in separate V8 isolates and don't
+ * share the main thread's class registrations, causing CreateInstance() to
+ * return null. Since the bottleneck is I/O (embedding API calls, vector DB
+ * upserts), not CPU, worker threads provide no benefit here.
+ */
+export declare class AsyncBatchTransform<TRecord = Record<string, unknown>, TContext = Record<string, unknown>, TResult = TRecord> extends Transform {
+    private readonly _batchSize;
+    private readonly _concurrencyLimit;
+    private readonly _context;
+    private readonly _processBatch;
+    private _buffer;
+    private _queue;
+    private _running;
+    private _drainResolve;
+    constructor(options: AsyncBatchTransformOptions<TRecord, TContext, TResult>);
+    private _next;
+    private _enqueue;
+    /** Returns a Promise that resolves when all queued tasks have completed */
+    private _waitForDrain;
+    _transform(chunk: TRecord, _encoding: BufferEncoding, callback: TransformCallback): void;
+    _flush(callback: TransformCallback): void;
+    private _processBatchAsync;
+}
+//# sourceMappingURL=AsyncBatchTransform.d.ts.map

package/dist/models/AsyncBatchTransform.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AsyncBatchTransform.d.ts","sourceRoot":"","sources":["../../src/models/AsyncBatchTransform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI;IACnE,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,+EAA+E;IAC/E,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,QAAQ,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;CAC3E,CAAC;AAEF;;;;;;;;;GASG;AACH,qBAAa,mBAAmB,CAC9B,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,OAAO,GAAG,OAAO,CACjB,SAAQ,SAAS;IACjB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA8D;IAE5F,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,aAAa,CAA6B;gBAEtC,OAAO,EAAE,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC;IAQ3E,OAAO,CAAC,KAAK;IAiBb,OAAO,CAAC,QAAQ;IAKhB,2EAA2E;IAC3E,OAAO,CAAC,aAAa;IASZ,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,iBAAiB,GAAG,IAAI;IASxF,MAAM,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI;YASpC,kBAAkB;CAUjC"}

package/dist/models/AsyncBatchTransform.js

@@ -0,0 +1,80 @@
+import { Transform } from 'node:stream';
+/**
+ * A Transform stream that batches records and processes them asynchronously
+ * in the main thread using a concurrency-limited queue.
+ *
+ * This replaces the worker_threads-based BatchWorker to solve the ClassFactory
+ * registration issue: worker threads run in separate V8 isolates and don't
+ * share the main thread's class registrations, causing CreateInstance() to
+ * return null. Since the bottleneck is I/O (embedding API calls, vector DB
+ * upserts), not CPU, worker threads provide no benefit here.
+ */
+export class AsyncBatchTransform extends Transform {
+    constructor(options) {
+        super({ objectMode: true });
+        this._buffer = [];
+        this._queue = [];
+        this._running = 0;
+        this._drainResolve = null;
+        this._batchSize = options.batchSize ?? 10;
+        this._concurrencyLimit = options.concurrencyLimit ?? 4;
+        this._context = options.context;
+        this._processBatch = options.processBatch;
+    }
+    _next() {
+        while (this._queue.length > 0 && this._running < this._concurrencyLimit) {
+            const task = this._queue.shift();
+            if (task) {
+                this._running++;
+                task().finally(() => {
+                    this._running--;
+                    this._next();
+                    if (this._running === 0 && this._queue.length === 0 && this._drainResolve) {
+                        this._drainResolve();
+                        this._drainResolve = null;
+                    }
+                });
+            }
+        }
+    }
+    _enqueue(task) {
+        this._queue.push(task);
+        this._next();
+    }
+    /** Returns a Promise that resolves when all queued tasks have completed */
+    _waitForDrain() {
+        if (this._running === 0 && this._queue.length === 0) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this._drainResolve = resolve;
+        });
+    }
+    _transform(chunk, _encoding, callback) {
+        this._buffer.push(chunk);
+        if (this._buffer.length >= this._batchSize) {
+            const batch = this._buffer.splice(0, this._batchSize);
+            this._enqueue(() => this._processBatchAsync(batch));
+        }
+        callback();
+    }
+    _flush(callback) {
+        if (this._buffer.length > 0) {
+            const remaining = this._buffer.splice(0);
+            this._enqueue(() => this._processBatchAsync(remaining));
+        }
+        this._waitForDrain().then(() => callback()).catch(callback);
+    }
+    async _processBatchAsync(batch) {
+        try {
+            const results = await this._processBatch(batch, this._context);
+            for (const result of results) {
+                this.push(result);
+            }
+        }
+        catch (error) {
+            this.emit('error', error);
+        }
+    }
+}
+//# sourceMappingURL=AsyncBatchTransform.js.map

package/dist/models/AsyncBatchTransform.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AsyncBatchTransform.js","sourceRoot":"","sources":["../../src/models/AsyncBatchTransform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAkBxC;;;;;;;;;GASG;AACH,MAAM,OAAO,mBAIX,SAAQ,SAAS;IAWjB,YAAY,OAA+D;QACzE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QANtB,YAAO,GAAc,EAAE,CAAC;QACxB,WAAM,GAA+B,EAAE,CAAC;QACxC,aAAQ,GAAG,CAAC,CAAC;QACb,kBAAa,GAAwB,IAAI,CAAC;QAIhD,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAmB,CAAC;QAC5C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAC5C,CAAC;IAEO,KAAK;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACxE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;oBAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,IAAI,CAAC,KAAK,EAAE,CAAC;oBACb,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;wBAC1E,IAAI,CAAC,aAAa,EAAE,CAAC;wBACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;oBAC5B,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,IAAyB;QACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,2EAA2E;IACnE,aAAa;QACnB,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;IAEQ,UAAU,CAAC,KAAc,EAAE,SAAyB,EAAE,QAA2B;QACxF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACtD,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,QAAQ,EAAE,CAAC;IACb,CAAC;IAEQ,MAAM,CAAC,QAA2B;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAgB;QAC/C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;CACF"}

package/dist/models/EntityDocumentSuggester.d.ts

@@ -0,0 +1,74 @@
+/**
+ * @fileoverview AI-powered Entity Document template suggestion engine.
+ *
+ * Uses the "Entity Document Suggestion" AI prompt to analyze entity schemas
+ * and suggest optimal Nunjucks templates for duplicate detection, search,
+ * or classification use cases.
+ *
+ * @module @memberjunction/ai-vector-sync
+ */
+import { UserInfo } from '@memberjunction/core';
+/** Supported use cases for document suggestion */
+export type DocumentSuggestionUseCase = 'duplicate detection' | 'search' | 'classification';
+/** A relationship descriptor passed to the AI prompt */
+export interface RelationshipDescriptor {
+    Name: string;
+    RelatedEntity: string;
+    Fields: string[];
+    ForeignKeyField: string;
+}
+/** A field descriptor passed to the AI prompt */
+export interface FieldDescriptor {
+    Name: string;
+    Type: string;
+    IsPrimaryKey: boolean;
+    IsUnique: boolean;
+    MaxLength: number | null;
+    AllowsNull: boolean;
+    Description: string;
+}
+/** The structured JSON response from the AI prompt */
+export interface DocumentSuggestionResult {
+    template: string;
+    selectedFields: string[];
+    selectedRelationships: {
+        name: string;
+        fields: string[];
+    }[];
+    potentialMatchThreshold: number;
+    absoluteMatchThreshold: number;
+    reasoning: string;
+}
+/** Full response from the suggestion engine */
+export interface SuggestDocumentResponse {
+    Success: boolean;
+    ErrorMessage: string;
+    Suggestion: DocumentSuggestionResult | null;
+}
+/**
+ * Suggests Entity Document templates by analyzing entity schemas with AI.
+ *
+ * Uses the `AIPromptRunner` directly (not via Actions) per CLAUDE.md design philosophy.
+ */
+export declare class EntityDocumentSuggester {
+    /**
+     * Analyze an entity's schema and suggest an Entity Document template.
+     *
+     * @param entityName - The name of the entity to analyze
+     * @param useCase - The intended use case (defaults to 'duplicate detection')
+     * @param contextUser - The authenticated user context
+     * @returns A suggestion with template, selected fields, and threshold recommendations
+     */
+    SuggestDocument(entityName: string, useCase: DocumentSuggestionUseCase, contextUser: UserInfo): Promise<SuggestDocumentResponse>;
+    /** Resolve entity metadata by name */
+    private resolveEntity;
+    /** Build field descriptors from entity metadata */
+    private buildFieldDescriptors;
+    /** Build relationship descriptors from entity metadata */
+    private buildRelationshipDescriptors;
+    /** Find the suggestion prompt from AIEngine */
+    private findSuggestionPrompt;
+    /** Execute the AI prompt and parse the response */
+    private executeSuggestionPrompt;
+}
+//# sourceMappingURL=EntityDocumentSuggester.d.ts.map

package/dist/models/EntityDocumentSuggester.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EntityDocumentSuggester.d.ts","sourceRoot":"","sources":["../../src/models/EntityDocumentSuggester.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAA6C,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAQ3F,kDAAkD;AAClD,MAAM,MAAM,yBAAyB,GAAG,qBAAqB,GAAG,QAAQ,GAAG,gBAAgB,CAAC;AAE5F,wDAAwD;AACxD,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CAC3B;AAED,iDAAiD;AACjD,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,sDAAsD;AACtD,MAAM,WAAW,wBAAwB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,qBAAqB,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC5D,uBAAuB,EAAE,MAAM,CAAC;IAChC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,+CAA+C;AAC/C,MAAM,WAAW,uBAAuB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,wBAAwB,GAAG,IAAI,CAAC;CAC/C;AAED;;;;GAIG;AACH,qBAAa,uBAAuB;IAEhC;;;;;;;OAOG;IACU,eAAe,CACxB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,yBAAyB,EAClC,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,uBAAuB,CAAC;IAiBnC,sCAAsC;IACtC,OAAO,CAAC,aAAa;IASrB,mDAAmD;IACnD,OAAO,CAAC,qBAAqB;IAY7B,0DAA0D;IAC1D,OAAO,CAAC,4BAA4B;IAqBpC,+CAA+C;IAC/C,OAAO,CAAC,oBAAoB;IAa5B,mDAAmD;YACrC,uBAAuB;CAmDxC"}