@memberjunction/content-autotagging 6.1.0-edge.0 → 6.1.0-edge.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Engine/generic/AutotagBaseEngine.d.ts +40 -5
- package/dist/Engine/generic/AutotagBaseEngine.d.ts.map +1 -1
- package/dist/Engine/generic/AutotagBaseEngine.js +171 -23
- package/dist/Engine/generic/AutotagBaseEngine.js.map +1 -1
- package/dist/Engine/generic/FieldPathResolver.d.ts +78 -0
- package/dist/Engine/generic/FieldPathResolver.d.ts.map +1 -0
- package/dist/Engine/generic/FieldPathResolver.js +212 -0
- package/dist/Engine/generic/FieldPathResolver.js.map +1 -0
- package/dist/Engine/index.d.ts +1 -0
- package/dist/Engine/index.d.ts.map +1 -1
- package/dist/Engine/index.js +1 -0
- package/dist/Engine/index.js.map +1 -1
- package/package.json +17 -17
|
@@ -462,7 +462,7 @@ export declare class AutotagBaseEngine extends BaseEngine<AutotagBaseEngine> {
|
|
|
462
462
|
* each item's storage config once and shares the item-level-vs-chunk decision between the
|
|
463
463
|
* vector id and the metadata so the two always agree.
|
|
464
464
|
*/
|
|
465
|
-
protected buildVectorRecords(allChunks: EmbeddingChunk[], vectors: number[][], tagMap: Map<string, string[]>, infra: ResolvedVectorInfrastructure): VectorRecord[];
|
|
465
|
+
protected buildVectorRecords(allChunks: EmbeddingChunk[], vectors: number[][], tagMap: Map<string, string[]>, infra: ResolvedVectorInfrastructure, directivesByItem?: Map<string, Record<string, unknown> | undefined>): VectorRecord[];
|
|
466
466
|
/**
|
|
467
467
|
* Build the single {@link VectorRecord} for one embedding chunk. Resolves the item's storage
|
|
468
468
|
* config, decides item-level-vs-chunk once, and shares that decision between the vector id and
|
|
@@ -477,18 +477,43 @@ export declare class AutotagBaseEngine extends BaseEngine<AutotagBaseEngine> {
|
|
|
477
477
|
* @param tags Resolved tag names for `chunk.item`, if any.
|
|
478
478
|
* @param contentItemEntity The 'MJ: Content Items' entity metadata, for display-field resolution.
|
|
479
479
|
* @param infra Resolved vector infrastructure (source of provider directives).
|
|
480
|
+
* @param directivesByItem Directives precomputed per item ({@link precomputeProviderDirectives});
|
|
481
|
+
* when absent, directives are built inline (and a driver rejection throws).
|
|
480
482
|
*/
|
|
481
|
-
protected buildVectorRecord(chunk: EmbeddingChunk, vector: number[], chunkCountForItem: number, tags: string[] | undefined, contentItemEntity: EntityInfo | undefined, infra: ResolvedVectorInfrastructure): VectorRecord;
|
|
483
|
+
protected buildVectorRecord(chunk: EmbeddingChunk, vector: number[], chunkCountForItem: number, tags: string[] | undefined, contentItemEntity: EntityInfo | undefined, infra: ResolvedVectorInfrastructure, directivesByItem?: Map<string, Record<string, unknown> | undefined>): VectorRecord;
|
|
482
484
|
/**
|
|
483
485
|
* Compute a vector record's per-record provider directives (e.g. Pinecone namespace) from the
|
|
484
486
|
* source content item and the index's ProviderConfig. Returns undefined when the index has no
|
|
485
487
|
* ProviderConfig — the common case — so no provider routing is applied and the base
|
|
486
488
|
* BuildProviderDirectives is not even invoked. The item's full field set (GetAll) is handed to
|
|
487
489
|
* the driver so it can read whatever field its config names (e.g. `namespaceField: 'OrganizationID'`).
|
|
488
|
-
*
|
|
490
|
+
* Directives are always derived from the parent ContentItem — the same source for a chunk's
|
|
489
491
|
* vector as for an item-level vector.
|
|
492
|
+
*
|
|
493
|
+
* When the driver declares source-record field paths (`GetSourceRecordFieldPaths` — e.g. a
|
|
494
|
+
* dotted path resolved from a related record), the caller pre-resolves the per-item values
|
|
495
|
+
* via {@link FieldPathResolver} and passes them in; each value is injected into the record
|
|
496
|
+
* under its full path key so the driver's plain `sourceRecord[path]` lookup works unchanged.
|
|
497
|
+
*
|
|
498
|
+
* MAY THROW: a driver is allowed to reject a record from `BuildProviderDirectives` (e.g. a
|
|
499
|
+
* mandatory routing value its config requires is missing). Batch callers go through
|
|
500
|
+
* {@link precomputeProviderDirectives}, which converts a throw into a per-record failure.
|
|
501
|
+
*/
|
|
502
|
+
protected buildProviderDirectives(item: MJContentItemEntity, infra: ResolvedVectorInfrastructure, resolvedFields?: Map<string, Record<string, unknown>>): Record<string, unknown> | undefined;
|
|
503
|
+
/**
|
|
504
|
+
* Resolve the source-record field paths the driver declares for this index, for a batch of
|
|
505
|
+
* items. Returns per-item bags of `{ pathKey: value }`, or undefined when the driver declares
|
|
506
|
+
* no paths. Deliberately generic — what a path's value MEANS (a Pinecone namespace, a shard
|
|
507
|
+
* key, a routing region...) is entirely the driver's business; the engine only resolves data.
|
|
490
508
|
*/
|
|
491
|
-
|
|
509
|
+
private resolveDriverFieldPaths;
|
|
510
|
+
/**
|
|
511
|
+
* Build directives for every item up front, converting a driver rejection (a throw from
|
|
512
|
+
* `BuildProviderDirectives`) into a per-record failure. Returns the directive map for the
|
|
513
|
+
* accepted items plus the rejected items — callers fail the rejected records individually
|
|
514
|
+
* (before any embedding spend) and proceed with the rest of the batch.
|
|
515
|
+
*/
|
|
516
|
+
private precomputeProviderDirectives;
|
|
492
517
|
/**
|
|
493
518
|
* Choose the vector-DB record id for one chunk based on the item's resolved config.
|
|
494
519
|
* - 'recordId' (default): the chunk's own id (chunkID), which is also the ContentItemChunk PK.
|
|
@@ -592,7 +617,10 @@ export declare class AutotagBaseEngine extends BaseEngine<AutotagBaseEngine> {
|
|
|
592
617
|
* Bounded per run by `maxItems` and per embed/upsert call by CHUNK_EMBED_SUBBATCH + rate-limited,
|
|
593
618
|
* so a large backlog drains over several runs rather than hammering the API or vector store.
|
|
594
619
|
* Meant to run out-of-band from live vectorization (on demand or scheduled). Best-effort per
|
|
595
|
-
* chunk — a failure leaves the row 'Pending' (retried next run) and never aborts the
|
|
620
|
+
* chunk — a transient failure leaves the row 'Pending' (retried next run) and never aborts the
|
|
621
|
+
* pass. Exception: a chunk whose record the vector driver REJECTS (a throw from
|
|
622
|
+
* `BuildProviderDirectives`, e.g. a mandatory routing value is missing) is marked 'Failed'
|
|
623
|
+
* (see {@link markChunkEmbedFailed}) so it cannot starve the oldest-first pending queue.
|
|
596
624
|
*
|
|
597
625
|
* @returns counts of chunks embedded, failed, and skipped (empty text / missing parent).
|
|
598
626
|
*/
|
|
@@ -607,6 +635,13 @@ export declare class AutotagBaseEngine extends BaseEngine<AutotagBaseEngine> {
|
|
|
607
635
|
* chunk → parent-item → infrastructure resolution.
|
|
608
636
|
*/
|
|
609
637
|
private embedChunksByInfrastructure;
|
|
638
|
+
/**
|
|
639
|
+
* Mark a pending chunk's embed as Failed — used when the vector provider rejects the record's
|
|
640
|
+
* directives. Deliberately NOT left 'Pending': the pending queue drains oldest-first, so rows
|
|
641
|
+
* that can never succeed without a data fix would permanently occupy the front of every future
|
|
642
|
+
* run's MaxItems window and starve resolvable chunks behind them. Best-effort save.
|
|
643
|
+
*/
|
|
644
|
+
private markChunkEmbedFailed;
|
|
610
645
|
/** Embed + upsert one infra group's chunks in bounded sub-batches, then stamp each row Complete. */
|
|
611
646
|
private embedChunkGroup;
|
|
612
647
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AutotagBaseEngine.d.ts","sourceRoot":"","sources":["../../../src/Engine/generic/AutotagBaseEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAA4B,iBAAiB,EAA6D,QAAQ,EAAuB,UAAU,EAAmB,MAAM,sBAAsB,CAAA;AAErN,OAAO,EACH,qBAAqB,EAAE,mBAAmB,EAAE,uBAAuB,EACnE,yBAAyB,EAAE,mBAAmB,EAAE,yBAAyB,EACzE,4BAA4B,EAA8B,sBAAsB,EAClD,8BAA8B,EAC5D,yDAAyD,EAG5D,MAAM,+BAA+B,CAAA;AACtC,OAAO,KAAK,EAAE,iDAAiD,EAAE,MAAM,+BAA+B,CAAA;AACtG,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAA;AAC3G,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAI3C,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAA;AAMxF,OAAO,EAAE,cAAc,EAAe,MAAM,oBAAoB,CAAA;AAKhE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAE5E,OAAO,EAEH,8BAA8B,EAEjC,MAAM,iCAAiC,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAgB,MAAM,6BAA6B,CAAA;AAKtF;;;GAGG;AACH,MAAM,WAAW,4BAA4B;IACzC,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yFAAyF;IACzF,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,iDAAiD,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAClH;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,iDAAiD,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAClH,wIAAwI;AACxI,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,iDAAiD,CAAC,gBAAgB,CAAC,CAAC,CAAC;AACpH,2HAA2H;AAC3H,MAAM,MAAM,yBAAyB,GAAG,WAAW,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACnC;AAaD;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,mBAAmB,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,mDAAmD;AACnD,MAAM,WAAW,eAAe;IAC5B,mFAAmF;IACnF,MAAM,EAAE,MAAM,CAAC;IACf,mFAAmF;IACnF,MAAM,EAAE,MAAM,CAAC;IACf,6FAA6F;IAC7F,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,oDAAoD;AACpD,MAAM,WAAW,eAAe;IAC5B,+FAA+F;IAC/F,QAAQ,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,MAAM,EAAE,MAAM,CAAC;IACf,6FAA6F;IAC7F,OAAO,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC5B,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,YAAY,EAAE,MAAM,EAAE,CAAC;CAC1B;AAaD;;;;GAIG;AACH,qBACa,iBAAkB,SAAQ,UAAU,CAAC,iBAAiB,CAAC;IAChE,WAAkB,QAAQ,IAAI,iBAAiB,CAE9C;IAED;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAqB;IAGjE,OAAO,CAAC,sBAAsB,CAAsC;IACpE,OAAO,CAAC,wBAAwB,CAAwC;IAExE,6CAA6C;IAC7C,OAAO,KAAK,QAAQ,GAA8E;IAElG,kEAAkE;IAClE,IAAW,YAAY,IAAI,mBAAmB,EAAE,CAAuC;IACvF,yEAAyE;IACzE,IAAW,kBAAkB,IAAI,yBAAyB,EAAE,CAA6C;IACzG,uEAAuE;IACvE,IAAW,gBAAgB,IAAI,uBAAuB,EAAE,CAA2C;IACnG,qDAAqD;IACrD,IAAW,qBAAqB,IAAI,4BAA4B,EAAE,CAAwC;IAC1G,wDAAwD;IACxD,IAAW,uBAAuB,IAAI,8BAA8B,EAAE,CAA0C;IAEnG,MAAM,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC;IAqBnH;;;OAGG;IACH;;;;;;;;;;;OAWG;IACU,4BAA4B,CACrC,YAAY,EAAE,mBAAmB,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC,EACxE,WAAW,EAAE,QAAQ,EACrB,UAAU,CAAC,EAAE,yBAAyB,EACtC,MAAM,CAAC,EAAE,yDAAyD,EAClE,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,KAAK,IAAI,GAC9E,OAAO,CAAC,IAAI,CAAC;IA8KhB;;OAEG;YACW,qBAAqB;IAkBnC;;OAEG;IACU,sBAAsB,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAe3G,2DAA2D;YAC7C,0BAA0B;IA4BxC,6DAA6D;YAC/C,8BAA8B;IA2B5C;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;;;OAKG;IACI,eAAe,EAAE,MAAM,GAAG,IAAI,CAAQ;IAE7C;;;;OAIG;IACI,cAAc,UAAS;IAE9B;;;;OAIG;IACI,yBAAyB,UAAS;IAEzC,+CAA+C;IACxC,cAAc,cAAoF;IACzG,2CAA2C;IACpC,oBAAoB,cAA2F;IACtH,2CAA2C;IACpC,mBAAmB,cAAiE;IAE3F;;;;;;;;;;;OAWG;IACU,wBAAwB,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B3E;;OAEG;IACI,qBAAqB,IAAI,IAAI;IAKpC;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAkB7B;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,uBAAuB;IAM/B;;;;;;;OAOG;YACW,8BAA8B;IAqD5C;;;;;;;;OAQG;YACW,kCAAkC;IA6EhD;;OAEG;IACH,OAAO,CAAC,eAAe;IA2BvB;;;;;OAKG;YACW,4BAA4B;IAmB1C;;;OAGG;IACH,OAAO,CAAC,6BAA6B;IAexB,+BAA+B,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;IAiC1H;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;;OAIG;IACU,4BAA4B,CACrC,MAAM,EAAE,wBAAwB,EAChC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,UAAU,CAAC;IA0DT,cAAc,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5E,wBAAwB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlG;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,mBAAmB,IAAI,MAAM;IAIvC;;;;;OAKG;cACa,sBAAsB,CAClC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,8BAA8B,GACxC,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;IAW3B;;;;;;OAMG;IACU,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAsBpF;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;;;;OAMG;IACI,qBAAqB,EAAE,CAAC,CAAC,GAAG,EAAE,sBAAsB,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,WAAW,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAQ;IAE9I;;;;;OAKG;IACI,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE,mBAAmB,EAAE,EAAE,cAAc,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAQ;IAEtJ;;;;;OAKG;IACU,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA8DrH;;;OAGG;IACU,iCAAiC,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAqC5G;;;OAGG;IACU,oBAAoB,CAAC,WAAW,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAQvH;;;OAGG;IACU,wBAAwB,CAAC,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAIrH,4BAA4B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAQhD,sBAAsB,CAAC,aAAa,EAAE,qBAAqB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,2BAA2B,CAAC,CAAC;IA2B5I,iCAAiC,CAAC,wBAAwB,EAAE,MAAM,GAAG,uBAAuB;IAa5F,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,2BAA2B;IAiBhF,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIrC,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAIhD;;OAEG;IACU,4BAA4B,CAAC,WAAW,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3E;;OAEG;IACU,2BAA2B,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBhG,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAYlG,wBAAwB,CAAC,mBAAmB,EAAE,MAAM,GAAG,MAAM;IAQ7D,kBAAkB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM;IAQjD,sBAAsB,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM;IAQzD,8BAA8B,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM;IAS7D,yBAAyB,CAAC,mBAAmB,EAAE,mBAAmB,GAAG,MAAM;IAOrE,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAMhD,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIlD,uBAAuB,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBtH;;OAEG;IACU,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAarG;;;;OAIG;IACU,uBAAuB,CAChC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,QAAQ,EACrB,MAAM,CAAC,EAAE,yDAAyD,GACnE,OAAO,CAAC,yBAAyB,CAAC;IA0BrC;;;;OAIG;IACU,iBAAiB,CAC1B,UAAU,EAAE,yBAAyB,EACrC,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,MAAM,GACnB,OAAO,CAAC,OAAO,CAAC;IAmBnB;;OAEG;IACU,yBAAyB,CAClC,UAAU,EAAE,yBAAyB,EACrC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,EAC5C,YAAY,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,IAAI,CAAC;IAShB;;OAEG;IACI,kBAAkB,CACrB,MAAM,CAAC,EAAE,yDAAyD,GACnE;QAAE,GAAG,EAAE,WAAW,CAAC;QAAC,SAAS,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,WAAW,CAAA;KAAE;IAmBzD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK7C,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWxC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAejE;;;;;;;;;;;;;;;OAeG;IACU,qBAAqB,CAC9B,KAAK,EAAE,mBAAmB,EAAE,EAC5B,WAAW,EAAE,QAAQ,EACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,EACvD,SAAS,GAAE,MAAqC,GACjD,OAAO,CAAC,eAAe,CAAC;IAqC3B;;;;;;;;;;;;OAYG;YACW,cAAc;IAuE5B;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAYhC;;;OAGG;YACW,mBAAmB;IAiBjC;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,CACxB,SAAS,EAAE,cAAc,EAAE,EAC3B,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAC7B,KAAK,EAAE,4BAA4B,GACpC,YAAY,EAAE;IAajB;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,iBAAiB,CACvB,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,MAAM,EAAE,EAChB,iBAAiB,EAAE,MAAM,EACzB,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,EAC1B,iBAAiB,EAAE,UAAU,GAAG,SAAS,EACzC,KAAK,EAAE,4BAA4B,GACpC,YAAY;IAef;;;;;;;;OAQG;IACH,SAAS,CAAC,uBAAuB,CAC7B,IAAI,EAAE,mBAAmB,EACzB,KAAK,EAAE,4BAA4B,GACpC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAOtC;;;;;;;;OAQG;IACH,SAAS,CAAC,oBAAoB,CAC1B,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,2BAA2B,EACnC,WAAW,EAAE,OAAO,GACrB,MAAM;IAST;;;;;;;;;;;;;;OAcG;YACW,uBAAuB;IAqCrC;;;;;;;;;;;OAWG;YACW,wBAAwB;IAsBtC;;;;;;;;OAQG;YACW,cAAc;IA+B5B;;;;;;;;;;;OAWG;YACW,sBAAsB;IA6CpC;;;;;;;;;;;;;;OAcG;IACU,kBAAkB,CAC3B,WAAW,EAAE,QAAQ,EACrB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAChC,OAAO,CAAC,eAAe,CAAC;IAuB3B,8FAA8F;YAChF,iBAAiB;IAgB/B,wGAAwG;YAC1F,oBAAoB;IAkBlC,gGAAgG;YAClF,2BAA2B;IA4BzC,iGAAiG;YACnF,eAAe;IAiB7B,uGAAuG;YACzF,kBAAkB;IAgBhC,iGAAiG;YACnF,gBAAgB;IAU9B;;;;;;;;;;;;;;OAcG;IACU,kBAAkB,CAC3B,WAAW,EAAE,QAAQ,EACrB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAChC,OAAO,CAAC,eAAe,CAAC;IAc3B,qFAAqF;YACvE,0BAA0B;IAgBxC;;;;OAIG;YACW,2BAA2B;IA8BzC,oGAAoG;YACtF,eAAe;IAgB7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAkB1B,oGAAoG;YACtF,yBAAyB;IAiDvC,qGAAqG;YACvF,iBAAiB;IAW/B;;;OAGG;YACW,mBAAmB;IAwBjC;;;OAGG;YACW,4BAA4B;IA4B1C;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IA2BpC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IAkBlC,8EAA8E;IAC9E,OAAO,CAAC,aAAa;IAMrB;;;OAGG;YACW,0BAA0B;IAcxC;;;OAGG;YACW,yBAAyB;IAavC;;OAEG;YACW,8BAA8B;IAS5C;;;;;OAKG;YACW,6BAA6B;IA8B3C;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAa3B,8EAA8E;IAC9E,OAAO,CAAC,kBAAkB;IAU1B,gEAAgE;IAChE,OAAO,CAAC,uBAAuB;IAU/B,2DAA2D;IAC3D,OAAO,CAAC,sBAAsB;IAU9B,uDAAuD;IACvD,OAAO,CAAC,mBAAmB;IAI3B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAIhC;;;;OAIG;IACH,SAAS,CAAC,8BAA8B,CAAC,IAAI,EAAE,mBAAmB,GAAG,2BAA2B;IAchG;;;;;;OAMG;IACH,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,2BAA2B,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO;IAI7F,yEAAyE;IACzE;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAEpD;;;OAGG;YACW,oBAAoB;IAiClC;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,mBAAmB,CACzB,KAAK,EAAE,cAAc,EACrB,WAAW,EAAE,OAAO,EACpB,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,EAC1B,MAAM,EAAE,2BAA2B,EACnC,iBAAiB,EAAE,UAAU,GAAG,SAAS,GAC1C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,EAAE,CAAC;IA8BvD;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAiBhC,mGAAmG;IACnG,OAAO,CAAC,kBAAkB;IAW1B,kGAAkG;IAClG,OAAO,CAAC,wBAAwB;IAahC;;;;;;OAMG;IACH,OAAO,CAAC,uBAAuB;IA4B/B,qGAAqG;IACrG,OAAO,CAAC,oBAAoB;IAwB5B,6FAA6F;IAC7F,OAAO,CAAC,8BAA8B;IAOtC,+FAA+F;IAC/F,OAAO,CAAC,qBAAqB;IAU7B,gGAAgG;IAChG,OAAO,CAAC,oBAAoB;IAU5B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAWvB,iEAAiE;YACnD,gBAAgB;IAyB9B;;;;OAIG;YACW,gCAAgC;IA2B9C;;;;;;;;;;;;;;;OAeG;IACU,wBAAwB,CAAC,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB7G;;;;;;;;;;;;;;;OAeG;IACU,qBAAqB,CAAC,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1G;;;;;;OAMG;IACU,gBAAgB,CAAC,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAOrG;;;;;;;;;;;OAWG;IACU,sBAAsB,CAC/B,WAAW,EAAE,mBAAmB,EAChC,WAAW,EAAE,QAAQ,EACrB,iBAAiB,UAAQ,GAC1B,OAAO,CAAC,IAAI,CAAC;IAYhB;;;;OAIG;YACW,uBAAuB;IAqFrC;;OAEG;YACW,qBAAqB;IAcnC;;;;;;OAMG;IACU,uBAAuB,CAChC,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,OAAO,GAAG,OAAO,GAAG,cAAc,EAC9C,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,OAAO,CAAC;IA2BnB;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAchC;;;;;;;;OAQG;YACW,mBAAmB;IAqBjC;;;;;;;;OAQG;YACW,gBAAgB;IAsB9B;;;;;;;;;;;;OAYG;YACW,gCAAgC;IAqC9C;;;;;;;;OAQG;YACW,mBAAmB;CAgBpC"}
|
|
1
|
+
{"version":3,"file":"AutotagBaseEngine.d.ts","sourceRoot":"","sources":["../../../src/Engine/generic/AutotagBaseEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAA4B,iBAAiB,EAA6D,QAAQ,EAAuB,UAAU,EAAmB,MAAM,sBAAsB,CAAA;AAErN,OAAO,EACH,qBAAqB,EAAE,mBAAmB,EAAE,uBAAuB,EACnE,yBAAyB,EAAE,mBAAmB,EAAE,yBAAyB,EACzE,4BAA4B,EAA8B,sBAAsB,EAClD,8BAA8B,EAC5D,yDAAyD,EAG5D,MAAM,+BAA+B,CAAA;AACtC,OAAO,KAAK,EAAE,iDAAiD,EAAE,MAAM,+BAA+B,CAAA;AACtG,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAA;AAC3G,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAI3C,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAA;AAOxF,OAAO,EAAE,cAAc,EAAe,MAAM,oBAAoB,CAAA;AAKhE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAE5E,OAAO,EAEH,8BAA8B,EAEjC,MAAM,iCAAiC,CAAA;AACxC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAgB,MAAM,6BAA6B,CAAA;AAKtF;;;GAGG;AACH,MAAM,WAAW,4BAA4B;IACzC,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yFAAyF;IACzF,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,iDAAiD,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAClH;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,iDAAiD,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAClH,wIAAwI;AACxI,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,iDAAiD,CAAC,gBAAgB,CAAC,CAAC,CAAC;AACpH,2HAA2H;AAC3H,MAAM,MAAM,yBAAyB,GAAG,WAAW,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACnC;AAaD;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,mBAAmB,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,mDAAmD;AACnD,MAAM,WAAW,eAAe;IAC5B,mFAAmF;IACnF,MAAM,EAAE,MAAM,CAAC;IACf,mFAAmF;IACnF,MAAM,EAAE,MAAM,CAAC;IACf,6FAA6F;IAC7F,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,oDAAoD;AACpD,MAAM,WAAW,eAAe;IAC5B,+FAA+F;IAC/F,QAAQ,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,MAAM,EAAE,MAAM,CAAC;IACf,6FAA6F;IAC7F,OAAO,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC5B,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,YAAY,EAAE,MAAM,EAAE,CAAC;CAC1B;AAaD;;;;GAIG;AACH,qBACa,iBAAkB,SAAQ,UAAU,CAAC,iBAAiB,CAAC;IAChE,WAAkB,QAAQ,IAAI,iBAAiB,CAE9C;IAED;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAqB;IAGjE,OAAO,CAAC,sBAAsB,CAAsC;IACpE,OAAO,CAAC,wBAAwB,CAAwC;IAExE,6CAA6C;IAC7C,OAAO,KAAK,QAAQ,GAA8E;IAElG,kEAAkE;IAClE,IAAW,YAAY,IAAI,mBAAmB,EAAE,CAAuC;IACvF,yEAAyE;IACzE,IAAW,kBAAkB,IAAI,yBAAyB,EAAE,CAA6C;IACzG,uEAAuE;IACvE,IAAW,gBAAgB,IAAI,uBAAuB,EAAE,CAA2C;IACnG,qDAAqD;IACrD,IAAW,qBAAqB,IAAI,4BAA4B,EAAE,CAAwC;IAC1G,wDAAwD;IACxD,IAAW,uBAAuB,IAAI,8BAA8B,EAAE,CAA0C;IAEnG,MAAM,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC;IAqBnH;;;OAGG;IACH;;;;;;;;;;;OAWG;IACU,4BAA4B,CACrC,YAAY,EAAE,mBAAmB,EAAE,GAAG,aAAa,CAAC,mBAAmB,CAAC,EACxE,WAAW,EAAE,QAAQ,EACrB,UAAU,CAAC,EAAE,yBAAyB,EACtC,MAAM,CAAC,EAAE,yDAAyD,EAClE,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,KAAK,IAAI,GAC9E,OAAO,CAAC,IAAI,CAAC;IA8KhB;;OAEG;YACW,qBAAqB;IAkBnC;;OAEG;IACU,sBAAsB,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAe3G,2DAA2D;YAC7C,0BAA0B;IA4BxC,6DAA6D;YAC/C,8BAA8B;IA2B5C;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;;;OAKG;IACI,eAAe,EAAE,MAAM,GAAG,IAAI,CAAQ;IAE7C;;;;OAIG;IACI,cAAc,UAAS;IAE9B;;;;OAIG;IACI,yBAAyB,UAAS;IAEzC,+CAA+C;IACxC,cAAc,cAAoF;IACzG,2CAA2C;IACpC,oBAAoB,cAA2F;IACtH,2CAA2C;IACpC,mBAAmB,cAAiE;IAE3F;;;;;;;;;;;OAWG;IACU,wBAAwB,CAAC,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B3E;;OAEG;IACI,qBAAqB,IAAI,IAAI;IAKpC;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAkB7B;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,uBAAuB;IAM/B;;;;;;;OAOG;YACW,8BAA8B;IAqD5C;;;;;;;;OAQG;YACW,kCAAkC;IA6EhD;;OAEG;IACH,OAAO,CAAC,eAAe;IA2BvB;;;;;OAKG;YACW,4BAA4B;IAmB1C;;;OAGG;IACH,OAAO,CAAC,6BAA6B;IAexB,+BAA+B,CAAC,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;IAiC1H;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;;OAIG;IACU,4BAA4B,CACrC,MAAM,EAAE,wBAAwB,EAChC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,UAAU,EACtB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,UAAU,CAAC;IA0DT,cAAc,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5E,wBAAwB,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAOlG;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,mBAAmB,IAAI,MAAM;IAIvC;;;;;OAKG;cACa,sBAAsB,CAClC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,8BAA8B,GACxC,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;IAW3B;;;;;;OAMG;IACU,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAsBpF;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;;;;OAMG;IACI,qBAAqB,EAAE,CAAC,CAAC,GAAG,EAAE,sBAAsB,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,WAAW,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAQ;IAE9I;;;;;OAKG;IACI,YAAY,EAAE,CAAC,CAAC,KAAK,EAAE,mBAAmB,EAAE,EAAE,cAAc,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAQ;IAEtJ;;;;;OAKG;IACU,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA8DrH;;;OAGG;IACU,iCAAiC,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAqC5G;;;OAGG;IACU,oBAAoB,CAAC,WAAW,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAQvH;;;OAGG;IACU,wBAAwB,CAAC,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAIrH,4BAA4B,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAQhD,sBAAsB,CAAC,aAAa,EAAE,qBAAqB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,2BAA2B,CAAC,CAAC;IA2B5I,iCAAiC,CAAC,wBAAwB,EAAE,MAAM,GAAG,uBAAuB;IAa5F,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,2BAA2B;IAiBhF,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIrC,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAIhD;;OAEG;IACU,4BAA4B,CAAC,WAAW,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3E;;OAEG;IACU,2BAA2B,CAAC,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBhG,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE;IAYlG,wBAAwB,CAAC,mBAAmB,EAAE,MAAM,GAAG,MAAM;IAQ7D,kBAAkB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM;IAQjD,sBAAsB,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM;IAQzD,8BAA8B,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM;IAS7D,yBAAyB,CAAC,mBAAmB,EAAE,mBAAmB,GAAG,MAAM;IAOrE,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAMhD,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIlD,uBAAuB,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBtH;;OAEG;IACU,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAarG;;;;OAIG;IACU,uBAAuB,CAChC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,QAAQ,EACrB,MAAM,CAAC,EAAE,yDAAyD,GACnE,OAAO,CAAC,yBAAyB,CAAC;IA0BrC;;;;OAIG;IACU,iBAAiB,CAC1B,UAAU,EAAE,yBAAyB,EACrC,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,MAAM,GACnB,OAAO,CAAC,OAAO,CAAC;IAmBnB;;OAEG;IACU,yBAAyB,CAClC,UAAU,EAAE,yBAAyB,EACrC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,EAC5C,YAAY,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,IAAI,CAAC;IAShB;;OAEG;IACI,kBAAkB,CACrB,MAAM,CAAC,EAAE,yDAAyD,GACnE;QAAE,GAAG,EAAE,WAAW,CAAC;QAAC,SAAS,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,WAAW,CAAA;KAAE;IAmBzD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK7C,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWxC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAejE;;;;;;;;;;;;;;;OAeG;IACU,qBAAqB,CAC9B,KAAK,EAAE,mBAAmB,EAAE,EAC5B,WAAW,EAAE,QAAQ,EACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,EACvD,SAAS,GAAE,MAAqC,GACjD,OAAO,CAAC,eAAe,CAAC;IAyC3B;;;;;;;;;;;;OAYG;YACW,cAAc;IAyF5B;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAYhC;;;OAGG;YACW,mBAAmB;IAiBjC;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,CACxB,SAAS,EAAE,cAAc,EAAE,EAC3B,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAC7B,KAAK,EAAE,4BAA4B,EACnC,gBAAgB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,GACpE,YAAY,EAAE;IAajB;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,iBAAiB,CACvB,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,MAAM,EAAE,EAChB,iBAAiB,EAAE,MAAM,EACzB,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,EAC1B,iBAAiB,EAAE,UAAU,GAAG,SAAS,EACzC,KAAK,EAAE,4BAA4B,EACnC,gBAAgB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,GACpE,YAAY;IAiBf;;;;;;;;;;;;;;;;;OAiBG;IACH,SAAS,CAAC,uBAAuB,CAC7B,IAAI,EAAE,mBAAmB,EACzB,KAAK,EAAE,4BAA4B,EACnC,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GACtD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAYtC;;;;;OAKG;YACW,uBAAuB;IAsBrC;;;;;OAKG;YACW,4BAA4B;IAmB1C;;;;;;;;OAQG;IACH,SAAS,CAAC,oBAAoB,CAC1B,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,2BAA2B,EACnC,WAAW,EAAE,OAAO,GACrB,MAAM;IAST;;;;;;;;;;;;;;OAcG;YACW,uBAAuB;IAqCrC;;;;;;;;;;;OAWG;YACW,wBAAwB;IAsBtC;;;;;;;;OAQG;YACW,cAAc;IA+B5B;;;;;;;;;;;OAWG;YACW,sBAAsB;IA6CpC;;;;;;;;;;;;;;OAcG;IACU,kBAAkB,CAC3B,WAAW,EAAE,QAAQ,EACrB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAChC,OAAO,CAAC,eAAe,CAAC;IAuB3B,8FAA8F;YAChF,iBAAiB;IAgB/B,wGAAwG;YAC1F,oBAAoB;IAkBlC,gGAAgG;YAClF,2BAA2B;IA6CzC,iGAAiG;YACnF,eAAe;IAmB7B,uGAAuG;YACzF,kBAAkB;IA8BhC,iGAAiG;YACnF,gBAAgB;IAU9B;;;;;;;;;;;;;;;;;OAiBG;IACU,kBAAkB,CAC3B,WAAW,EAAE,QAAQ,EACrB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAChC,OAAO,CAAC,eAAe,CAAC;IAc3B,qFAAqF;YACvE,0BAA0B;IAgBxC;;;;OAIG;YACW,2BAA2B;IAiDzC;;;;;OAKG;YACW,oBAAoB;IASlC,oGAAoG;YACtF,eAAe;IAiB7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAkB1B,oGAAoG;YACtF,yBAAyB;IAoDvC,qGAAqG;YACvF,iBAAiB;IAW/B;;;OAGG;YACW,mBAAmB;IAwBjC;;;OAGG;YACW,4BAA4B;IA4B1C;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IA2BpC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IAkBlC,8EAA8E;IAC9E,OAAO,CAAC,aAAa;IAMrB;;;OAGG;YACW,0BAA0B;IAcxC;;;OAGG;YACW,yBAAyB;IAavC;;OAEG;YACW,8BAA8B;IAS5C;;;;;OAKG;YACW,6BAA6B;IA8B3C;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAa3B,8EAA8E;IAC9E,OAAO,CAAC,kBAAkB;IAU1B,gEAAgE;IAChE,OAAO,CAAC,uBAAuB;IAU/B,2DAA2D;IAC3D,OAAO,CAAC,sBAAsB;IAU9B,uDAAuD;IACvD,OAAO,CAAC,mBAAmB;IAI3B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAIhC;;;;OAIG;IACH,SAAS,CAAC,8BAA8B,CAAC,IAAI,EAAE,mBAAmB,GAAG,2BAA2B;IAchG;;;;;;OAMG;IACH,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,2BAA2B,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO;IAI7F,yEAAyE;IACzE;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAEpD;;;OAGG;YACW,oBAAoB;IAiClC;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,mBAAmB,CACzB,KAAK,EAAE,cAAc,EACrB,WAAW,EAAE,OAAO,EACpB,IAAI,EAAE,MAAM,EAAE,GAAG,SAAS,EAC1B,MAAM,EAAE,2BAA2B,EACnC,iBAAiB,EAAE,UAAU,GAAG,SAAS,GAC1C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,EAAE,CAAC;IA8BvD;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAiBhC,mGAAmG;IACnG,OAAO,CAAC,kBAAkB;IAW1B,kGAAkG;IAClG,OAAO,CAAC,wBAAwB;IAahC;;;;;;OAMG;IACH,OAAO,CAAC,uBAAuB;IA4B/B,qGAAqG;IACrG,OAAO,CAAC,oBAAoB;IAwB5B,6FAA6F;IAC7F,OAAO,CAAC,8BAA8B;IAOtC,+FAA+F;IAC/F,OAAO,CAAC,qBAAqB;IAU7B,gGAAgG;IAChG,OAAO,CAAC,oBAAoB;IAU5B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAWvB,iEAAiE;YACnD,gBAAgB;IAyB9B;;;;OAIG;YACW,gCAAgC;IA2B9C;;;;;;;;;;;;;;;OAeG;IACU,wBAAwB,CAAC,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB7G;;;;;;;;;;;;;;;OAeG;IACU,qBAAqB,CAAC,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1G;;;;;;OAMG;IACU,gBAAgB,CAAC,WAAW,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAOrG;;;;;;;;;;;OAWG;IACU,sBAAsB,CAC/B,WAAW,EAAE,mBAAmB,EAChC,WAAW,EAAE,QAAQ,EACrB,iBAAiB,UAAQ,GAC1B,OAAO,CAAC,IAAI,CAAC;IAYhB;;;;OAIG;YACW,uBAAuB;IAqFrC;;OAEG;YACW,qBAAqB;IAcnC;;;;;;OAMG;IACU,uBAAuB,CAChC,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,OAAO,GAAG,OAAO,GAAG,cAAc,EAC9C,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,OAAO,CAAC;IA2BnB;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAchC;;;;;;;;OAQG;YACW,mBAAmB;IAqBjC;;;;;;;;OAQG;YACW,gBAAgB;IAsB9B;;;;;;;;;;;;OAYG;YACW,gCAAgC;IAqC9C;;;;;;;;OAQG;YACW,mBAAmB;CAgBpC"}
|
|
@@ -14,6 +14,7 @@ import officeparser from 'officeparser';
|
|
|
14
14
|
import * as fs from 'fs';
|
|
15
15
|
import { ProcessRunParams, ContentItemProcessParams } from './process.types.js';
|
|
16
16
|
import { ClassificationContextResolver } from './ClassificationContextResolver.js';
|
|
17
|
+
import { FieldPathResolver } from './FieldPathResolver.js';
|
|
17
18
|
import { toZonedTime } from 'date-fns-tz';
|
|
18
19
|
import axios from 'axios';
|
|
19
20
|
import * as cheerio from 'cheerio';
|
|
@@ -1296,9 +1297,12 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1296
1297
|
let vectorized = 0;
|
|
1297
1298
|
let processed = 0;
|
|
1298
1299
|
const allPromptRunIDs = [];
|
|
1300
|
+
// One resolver per pass: dotted field paths declared by the vector driver
|
|
1301
|
+
// ('<FKField>.<Field>') are batched and cached across every infra group below.
|
|
1302
|
+
const pathResolver = new FieldPathResolver(this.ProviderToUse, contextUser, 'MJ: Content Items');
|
|
1299
1303
|
for (const [groupKey, groupItems] of groups) {
|
|
1300
1304
|
const infra = await this.resolveGroupInfrastructure(groupKey, contextUser);
|
|
1301
|
-
const groupResult = await this.vectorizeGroup(groupItems, infra, tagMap, batchSize, contextUser, (batchProcessed) => {
|
|
1305
|
+
const groupResult = await this.vectorizeGroup(groupItems, infra, tagMap, batchSize, contextUser, pathResolver, (batchProcessed) => {
|
|
1302
1306
|
processed += batchProcessed;
|
|
1303
1307
|
onProgress?.(Math.min(processed, eligible.length), eligible.length);
|
|
1304
1308
|
});
|
|
@@ -1321,10 +1325,26 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1321
1325
|
* @param onBatchComplete - callback invoked after each batch with item count
|
|
1322
1326
|
* @returns count of vectorized items and collected AIPromptRun IDs
|
|
1323
1327
|
*/
|
|
1324
|
-
async vectorizeGroup(items, infra, tagMap, batchSize, contextUser, onBatchComplete) {
|
|
1328
|
+
async vectorizeGroup(items, infra, tagMap, batchSize, contextUser, pathResolver, onBatchComplete) {
|
|
1325
1329
|
let vectorized = 0;
|
|
1326
1330
|
const promptRunIDs = [];
|
|
1327
1331
|
const modelRunner = new AIModelRunner();
|
|
1332
|
+
// Provider directives are built for every item BEFORE any embedding spend. A driver may
|
|
1333
|
+
// reject a record from BuildProviderDirectives (e.g. a mandatory routing value its config
|
|
1334
|
+
// requires — such as a tenant-isolation field — is missing); rejected items are marked
|
|
1335
|
+
// Failed individually and excluded so they never poison the rest of the group. Recovery
|
|
1336
|
+
// after fixing the data: reset the item to Pending or use ForceReprocess.
|
|
1337
|
+
const { directives, rejected } = await this.precomputeProviderDirectives(items, infra, pathResolver);
|
|
1338
|
+
if (rejected.length > 0) {
|
|
1339
|
+
LogError(`VectorizeContentItems: ${rejected.length} item(s) rejected by the vector provider's directive policy — marked Failed, remaining items proceed`);
|
|
1340
|
+
await this.updateEmbeddingStatusBatch(rejected, 'Failed', contextUser);
|
|
1341
|
+
onBatchComplete(rejected.length);
|
|
1342
|
+
const rejectedIDs = new Set(rejected.map(i => NormalizeUUID(i.ID)));
|
|
1343
|
+
items = items.filter(i => !rejectedIDs.has(NormalizeUUID(i.ID)));
|
|
1344
|
+
}
|
|
1345
|
+
if (items.length === 0) {
|
|
1346
|
+
return { vectorized, promptRunIDs };
|
|
1347
|
+
}
|
|
1328
1348
|
// Mark every item in this infra group as Processing up front so dashboards
|
|
1329
1349
|
// reflect in-flight state. Each batch will transition its items to Complete
|
|
1330
1350
|
// or Failed as outcomes are known.
|
|
@@ -1357,7 +1377,7 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1357
1377
|
if (runResult.PromptRunID) {
|
|
1358
1378
|
promptRunIDs.push(runResult.PromptRunID);
|
|
1359
1379
|
}
|
|
1360
|
-
const records = this.buildVectorRecords(allChunks, runResult.Vectors, tagMap, infra);
|
|
1380
|
+
const records = this.buildVectorRecords(allChunks, runResult.Vectors, tagMap, infra, directives);
|
|
1361
1381
|
const batchSuccess = await this.upsertVectorRecords(records, infra);
|
|
1362
1382
|
if (batchSuccess) {
|
|
1363
1383
|
vectorized += batch.length;
|
|
@@ -1410,7 +1430,7 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1410
1430
|
* each item's storage config once and shares the item-level-vs-chunk decision between the
|
|
1411
1431
|
* vector id and the metadata so the two always agree.
|
|
1412
1432
|
*/
|
|
1413
|
-
buildVectorRecords(allChunks, vectors, tagMap, infra) {
|
|
1433
|
+
buildVectorRecords(allChunks, vectors, tagMap, infra, directivesByItem) {
|
|
1414
1434
|
const countByItem = new Map();
|
|
1415
1435
|
for (const c of allChunks) {
|
|
1416
1436
|
countByItem.set(c.item.ID, (countByItem.get(c.item.ID) ?? 0) + 1);
|
|
@@ -1418,7 +1438,7 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1418
1438
|
// The ContentItem entity metadata drives config-selected display fields (types, MaxLength,
|
|
1419
1439
|
// eligibility). Resolved once per batch — every chunk shares the same source entity.
|
|
1420
1440
|
const contentItemEntity = this.ProviderToUse.EntityByName('MJ: Content Items');
|
|
1421
|
-
return allChunks.map((chunk, idx) => this.buildVectorRecord(chunk, vectors[idx], countByItem.get(chunk.item.ID) ?? 1, tagMap.get(chunk.item.ID), contentItemEntity, infra));
|
|
1441
|
+
return allChunks.map((chunk, idx) => this.buildVectorRecord(chunk, vectors[idx], countByItem.get(chunk.item.ID) ?? 1, tagMap.get(chunk.item.ID), contentItemEntity, infra, directivesByItem));
|
|
1422
1442
|
}
|
|
1423
1443
|
/**
|
|
1424
1444
|
* Build the single {@link VectorRecord} for one embedding chunk. Resolves the item's storage
|
|
@@ -1434,8 +1454,10 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1434
1454
|
* @param tags Resolved tag names for `chunk.item`, if any.
|
|
1435
1455
|
* @param contentItemEntity The 'MJ: Content Items' entity metadata, for display-field resolution.
|
|
1436
1456
|
* @param infra Resolved vector infrastructure (source of provider directives).
|
|
1457
|
+
* @param directivesByItem Directives precomputed per item ({@link precomputeProviderDirectives});
|
|
1458
|
+
* when absent, directives are built inline (and a driver rejection throws).
|
|
1437
1459
|
*/
|
|
1438
|
-
buildVectorRecord(chunk, vector, chunkCountForItem, tags, contentItemEntity, infra) {
|
|
1460
|
+
buildVectorRecord(chunk, vector, chunkCountForItem, tags, contentItemEntity, infra, directivesByItem) {
|
|
1439
1461
|
const config = this.resolveItemVectorStorageConfig(chunk.item);
|
|
1440
1462
|
const itemLevel = this.isItemLevelVector(config, chunkCountForItem);
|
|
1441
1463
|
const record = {
|
|
@@ -1443,7 +1465,9 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1443
1465
|
values: vector,
|
|
1444
1466
|
metadata: this.buildVectorMetadata(chunk, itemLevel, tags, config, contentItemEntity)
|
|
1445
1467
|
};
|
|
1446
|
-
const directives =
|
|
1468
|
+
const directives = directivesByItem
|
|
1469
|
+
? directivesByItem.get(NormalizeUUID(chunk.item.ID))
|
|
1470
|
+
: this.buildProviderDirectives(chunk.item, infra);
|
|
1447
1471
|
if (directives) {
|
|
1448
1472
|
record.providerTemporaryDirectives = directives;
|
|
1449
1473
|
}
|
|
@@ -1455,14 +1479,73 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1455
1479
|
* ProviderConfig — the common case — so no provider routing is applied and the base
|
|
1456
1480
|
* BuildProviderDirectives is not even invoked. The item's full field set (GetAll) is handed to
|
|
1457
1481
|
* the driver so it can read whatever field its config names (e.g. `namespaceField: 'OrganizationID'`).
|
|
1458
|
-
*
|
|
1482
|
+
* Directives are always derived from the parent ContentItem — the same source for a chunk's
|
|
1459
1483
|
* vector as for an item-level vector.
|
|
1484
|
+
*
|
|
1485
|
+
* When the driver declares source-record field paths (`GetSourceRecordFieldPaths` — e.g. a
|
|
1486
|
+
* dotted path resolved from a related record), the caller pre-resolves the per-item values
|
|
1487
|
+
* via {@link FieldPathResolver} and passes them in; each value is injected into the record
|
|
1488
|
+
* under its full path key so the driver's plain `sourceRecord[path]` lookup works unchanged.
|
|
1489
|
+
*
|
|
1490
|
+
* MAY THROW: a driver is allowed to reject a record from `BuildProviderDirectives` (e.g. a
|
|
1491
|
+
* mandatory routing value its config requires is missing). Batch callers go through
|
|
1492
|
+
* {@link precomputeProviderDirectives}, which converts a throw into a per-record failure.
|
|
1460
1493
|
*/
|
|
1461
|
-
buildProviderDirectives(item, infra) {
|
|
1494
|
+
buildProviderDirectives(item, infra, resolvedFields) {
|
|
1462
1495
|
if (!infra.providerConfig) {
|
|
1463
1496
|
return undefined;
|
|
1464
1497
|
}
|
|
1465
|
-
|
|
1498
|
+
let sourceRecord = item.GetAll();
|
|
1499
|
+
const extra = resolvedFields?.get(NormalizeUUID(item.ID));
|
|
1500
|
+
if (extra) {
|
|
1501
|
+
sourceRecord = { ...sourceRecord, ...extra };
|
|
1502
|
+
}
|
|
1503
|
+
return infra.vectorDB.BuildProviderDirectives(sourceRecord, infra.providerConfig);
|
|
1504
|
+
}
|
|
1505
|
+
/**
|
|
1506
|
+
* Resolve the source-record field paths the driver declares for this index, for a batch of
|
|
1507
|
+
* items. Returns per-item bags of `{ pathKey: value }`, or undefined when the driver declares
|
|
1508
|
+
* no paths. Deliberately generic — what a path's value MEANS (a Pinecone namespace, a shard
|
|
1509
|
+
* key, a routing region...) is entirely the driver's business; the engine only resolves data.
|
|
1510
|
+
*/
|
|
1511
|
+
async resolveDriverFieldPaths(items, infra, resolver) {
|
|
1512
|
+
if (!infra.providerConfig)
|
|
1513
|
+
return undefined;
|
|
1514
|
+
const paths = infra.vectorDB.GetSourceRecordFieldPaths(infra.providerConfig);
|
|
1515
|
+
if (!paths || paths.length === 0)
|
|
1516
|
+
return undefined;
|
|
1517
|
+
const byItem = new Map();
|
|
1518
|
+
for (const path of paths) {
|
|
1519
|
+
const values = await resolver.ResolveForItems(items, path);
|
|
1520
|
+
for (const item of items) {
|
|
1521
|
+
const key = NormalizeUUID(item.ID);
|
|
1522
|
+
const bag = byItem.get(key) ?? {};
|
|
1523
|
+
bag[path] = values.get(key);
|
|
1524
|
+
byItem.set(key, bag);
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
return byItem;
|
|
1528
|
+
}
|
|
1529
|
+
/**
|
|
1530
|
+
* Build directives for every item up front, converting a driver rejection (a throw from
|
|
1531
|
+
* `BuildProviderDirectives`) into a per-record failure. Returns the directive map for the
|
|
1532
|
+
* accepted items plus the rejected items — callers fail the rejected records individually
|
|
1533
|
+
* (before any embedding spend) and proceed with the rest of the batch.
|
|
1534
|
+
*/
|
|
1535
|
+
async precomputeProviderDirectives(items, infra, resolver) {
|
|
1536
|
+
const directives = new Map();
|
|
1537
|
+
const rejected = [];
|
|
1538
|
+
const resolvedFields = await this.resolveDriverFieldPaths(items, infra, resolver);
|
|
1539
|
+
for (const item of items) {
|
|
1540
|
+
try {
|
|
1541
|
+
directives.set(NormalizeUUID(item.ID), this.buildProviderDirectives(item, infra, resolvedFields));
|
|
1542
|
+
}
|
|
1543
|
+
catch (e) {
|
|
1544
|
+
LogError(`Vector provider rejected record ${item.ID}: ${e instanceof Error ? e.message : String(e)}`);
|
|
1545
|
+
rejected.push(item);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
return { directives, rejected };
|
|
1466
1549
|
}
|
|
1467
1550
|
/**
|
|
1468
1551
|
* Choose the vector-DB record id for one chunk based on the item's resolved config.
|
|
@@ -1718,11 +1801,13 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1718
1801
|
stats.failed += toPurge.length;
|
|
1719
1802
|
return;
|
|
1720
1803
|
}
|
|
1804
|
+
const itemById = new Map(items.map(i => [NormalizeUUID(i.ID), i]));
|
|
1721
1805
|
const { sourceMap, typeMap } = await this.loadContentSourceAndTypeMaps(items, contextUser);
|
|
1722
1806
|
const itemGroups = this.groupItemsByInfrastructure(items, sourceMap, typeMap);
|
|
1807
|
+
const pathResolver = new FieldPathResolver(this.ProviderToUse, contextUser, 'MJ: Content Items');
|
|
1723
1808
|
for (const [groupKey, groupItems] of itemGroups) {
|
|
1724
1809
|
const groupItemIDs = new Set(groupItems.map(i => NormalizeUUID(i.ID)));
|
|
1725
|
-
|
|
1810
|
+
let groupChunks = toPurge.filter(c => groupItemIDs.has(NormalizeUUID(c.ContentItemID)));
|
|
1726
1811
|
if (groupChunks.length === 0)
|
|
1727
1812
|
continue;
|
|
1728
1813
|
let infra;
|
|
@@ -1734,14 +1819,28 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1734
1819
|
stats.failed += groupChunks.length;
|
|
1735
1820
|
continue;
|
|
1736
1821
|
}
|
|
1737
|
-
|
|
1822
|
+
// Deletes carry the same per-record directives as the original upsert so drivers can
|
|
1823
|
+
// route each removal correctly. Chunks whose parent the driver rejects can't have
|
|
1824
|
+
// their vectors targeted — they are left 'Pending' (counted failed) and retried after
|
|
1825
|
+
// the data is fixed; the rest of the group proceeds.
|
|
1826
|
+
const { directives, rejected } = await this.precomputeProviderDirectives(groupItems, infra, pathResolver);
|
|
1827
|
+
if (rejected.length > 0) {
|
|
1828
|
+
const rejectedIDs = new Set(rejected.map(i => NormalizeUUID(i.ID)));
|
|
1829
|
+
const blocked = groupChunks.filter(c => rejectedIDs.has(NormalizeUUID(c.ContentItemID)));
|
|
1830
|
+
LogError(`PurgeDeletedChunks: ${blocked.length} chunk(s) rejected by the vector provider's directive policy — cannot target their vectors, left Pending`);
|
|
1831
|
+
stats.failed += blocked.length;
|
|
1832
|
+
groupChunks = groupChunks.filter(c => !rejectedIDs.has(NormalizeUUID(c.ContentItemID)));
|
|
1833
|
+
if (groupChunks.length === 0)
|
|
1834
|
+
continue;
|
|
1835
|
+
}
|
|
1836
|
+
await this.purgeChunkGroup(groupChunks, infra, stats, itemById, directives);
|
|
1738
1837
|
}
|
|
1739
1838
|
}
|
|
1740
1839
|
/** Delete one infra group's chunk vectors in bounded sub-batches, then mark the rows Deleted. */
|
|
1741
|
-
async purgeChunkGroup(groupChunks, infra, stats) {
|
|
1840
|
+
async purgeChunkGroup(groupChunks, infra, stats, itemById, directivesByItem) {
|
|
1742
1841
|
for (let i = 0; i < groupChunks.length; i += PURGE_VECTORDB_SUBBATCH) {
|
|
1743
1842
|
const batch = groupChunks.slice(i, i + PURGE_VECTORDB_SUBBATCH);
|
|
1744
|
-
if (await this.deleteChunkVectors(batch, infra)) {
|
|
1843
|
+
if (await this.deleteChunkVectors(batch, infra, itemById, directivesByItem)) {
|
|
1745
1844
|
for (const c of batch) {
|
|
1746
1845
|
if (await this.markChunkDeleted(c))
|
|
1747
1846
|
stats.purged++;
|
|
@@ -1755,8 +1854,20 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1755
1854
|
}
|
|
1756
1855
|
}
|
|
1757
1856
|
/** Remove a batch of chunk vectors from the vector DB (rate-limited). Returns whether it succeeded. */
|
|
1758
|
-
async deleteChunkVectors(batch, infra) {
|
|
1759
|
-
|
|
1857
|
+
async deleteChunkVectors(batch, infra, itemById, directivesByItem) {
|
|
1858
|
+
// Deletes carry the same per-record provider directives as the original upsert so
|
|
1859
|
+
// drivers can route each removal correctly (e.g. to the partition the vector lives in).
|
|
1860
|
+
const records = batch.map(c => {
|
|
1861
|
+
const record = { id: c.VectorRecordID, values: [], metadata: {} };
|
|
1862
|
+
const item = itemById.get(NormalizeUUID(c.ContentItemID));
|
|
1863
|
+
const directives = directivesByItem
|
|
1864
|
+
? directivesByItem.get(NormalizeUUID(c.ContentItemID))
|
|
1865
|
+
: (item ? this.buildProviderDirectives(item, infra) : undefined);
|
|
1866
|
+
if (directives) {
|
|
1867
|
+
record.providerTemporaryDirectives = directives;
|
|
1868
|
+
}
|
|
1869
|
+
return record;
|
|
1870
|
+
});
|
|
1760
1871
|
await this.VectorDBRateLimiter.Acquire();
|
|
1761
1872
|
try {
|
|
1762
1873
|
const resp = await infra.vectorDB.DeleteRecords(records, infra.indexName);
|
|
@@ -1790,7 +1901,10 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1790
1901
|
* Bounded per run by `maxItems` and per embed/upsert call by CHUNK_EMBED_SUBBATCH + rate-limited,
|
|
1791
1902
|
* so a large backlog drains over several runs rather than hammering the API or vector store.
|
|
1792
1903
|
* Meant to run out-of-band from live vectorization (on demand or scheduled). Best-effort per
|
|
1793
|
-
* chunk — a failure leaves the row 'Pending' (retried next run) and never aborts the
|
|
1904
|
+
* chunk — a transient failure leaves the row 'Pending' (retried next run) and never aborts the
|
|
1905
|
+
* pass. Exception: a chunk whose record the vector driver REJECTS (a throw from
|
|
1906
|
+
* `BuildProviderDirectives`, e.g. a mandatory routing value is missing) is marked 'Failed'
|
|
1907
|
+
* (see {@link markChunkEmbedFailed}) so it cannot starve the oldest-first pending queue.
|
|
1794
1908
|
*
|
|
1795
1909
|
* @returns counts of chunks embedded, failed, and skipped (empty text / missing parent).
|
|
1796
1910
|
*/
|
|
@@ -1836,9 +1950,10 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1836
1950
|
const { sourceMap, typeMap } = await this.loadContentSourceAndTypeMaps(items, contextUser);
|
|
1837
1951
|
const itemGroups = this.groupItemsByInfrastructure(items, sourceMap, typeMap);
|
|
1838
1952
|
const tagMap = await this.loadTagsForItems(items, contextUser);
|
|
1953
|
+
const pathResolver = new FieldPathResolver(this.ProviderToUse, contextUser, 'MJ: Content Items');
|
|
1839
1954
|
for (const [groupKey, groupItems] of itemGroups) {
|
|
1840
1955
|
const groupItemIDs = new Set(groupItems.map(i => NormalizeUUID(i.ID)));
|
|
1841
|
-
|
|
1956
|
+
let groupChunks = chunks.filter(c => groupItemIDs.has(NormalizeUUID(c.ContentItemID)));
|
|
1842
1957
|
if (groupChunks.length === 0)
|
|
1843
1958
|
continue;
|
|
1844
1959
|
let infra;
|
|
@@ -1850,17 +1965,48 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1850
1965
|
stats.failed += groupChunks.length;
|
|
1851
1966
|
continue;
|
|
1852
1967
|
}
|
|
1853
|
-
|
|
1968
|
+
// Provider directives are built per parent item BEFORE any embedding spend. Chunks
|
|
1969
|
+
// whose parent the driver rejects (see precomputeProviderDirectives) are marked Failed
|
|
1970
|
+
// individually; the rest of the group proceeds. Recovery: fix the data, reset the
|
|
1971
|
+
// rows to Pending.
|
|
1972
|
+
const { directives, rejected } = await this.precomputeProviderDirectives(groupItems, infra, pathResolver);
|
|
1973
|
+
if (rejected.length > 0) {
|
|
1974
|
+
const rejectedIDs = new Set(rejected.map(i => NormalizeUUID(i.ID)));
|
|
1975
|
+
const blocked = groupChunks.filter(c => rejectedIDs.has(NormalizeUUID(c.ContentItemID)));
|
|
1976
|
+
LogError(`EmbedPendingChunks: ${blocked.length} chunk(s) rejected by the vector provider's directive policy — marked Failed, remaining chunks proceed`);
|
|
1977
|
+
for (const c of blocked) {
|
|
1978
|
+
await this.markChunkEmbedFailed(c);
|
|
1979
|
+
stats.failed++;
|
|
1980
|
+
}
|
|
1981
|
+
groupChunks = groupChunks.filter(c => !rejectedIDs.has(NormalizeUUID(c.ContentItemID)));
|
|
1982
|
+
if (groupChunks.length === 0)
|
|
1983
|
+
continue;
|
|
1984
|
+
}
|
|
1985
|
+
await this.embedChunkGroup(groupChunks, itemById, infra, tagMap, contextUser, stats, directives);
|
|
1854
1986
|
}
|
|
1855
1987
|
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Mark a pending chunk's embed as Failed — used when the vector provider rejects the record's
|
|
1990
|
+
* directives. Deliberately NOT left 'Pending': the pending queue drains oldest-first, so rows
|
|
1991
|
+
* that can never succeed without a data fix would permanently occupy the front of every future
|
|
1992
|
+
* run's MaxItems window and starve resolvable chunks behind them. Best-effort save.
|
|
1993
|
+
*/
|
|
1994
|
+
async markChunkEmbedFailed(chunk) {
|
|
1995
|
+
chunk.EmbeddingStatus = 'Failed';
|
|
1996
|
+
const saved = await chunk.Save();
|
|
1997
|
+
if (!saved) {
|
|
1998
|
+
LogError(`EmbedPendingChunks: failed to mark chunk ${chunk.ID} Failed: ${chunk.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
1999
|
+
}
|
|
2000
|
+
return saved;
|
|
2001
|
+
}
|
|
1856
2002
|
/** Embed + upsert one infra group's chunks in bounded sub-batches, then stamp each row Complete. */
|
|
1857
|
-
async embedChunkGroup(groupChunks, itemById, infra, tagMap, contextUser, stats) {
|
|
2003
|
+
async embedChunkGroup(groupChunks, itemById, infra, tagMap, contextUser, stats, directivesByItem) {
|
|
1858
2004
|
for (let i = 0; i < groupChunks.length; i += CHUNK_EMBED_SUBBATCH) {
|
|
1859
2005
|
const batch = groupChunks.slice(i, i + CHUNK_EMBED_SUBBATCH);
|
|
1860
2006
|
const embeddable = this.toEmbeddableChunks(batch, itemById, stats);
|
|
1861
2007
|
if (embeddable.length === 0)
|
|
1862
2008
|
continue;
|
|
1863
|
-
await this.embedAndPersistChunkBatch(embeddable, infra, tagMap, contextUser, stats);
|
|
2009
|
+
await this.embedAndPersistChunkBatch(embeddable, infra, tagMap, contextUser, stats, directivesByItem);
|
|
1864
2010
|
}
|
|
1865
2011
|
}
|
|
1866
2012
|
/**
|
|
@@ -1882,7 +2028,7 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1882
2028
|
return embeddable;
|
|
1883
2029
|
}
|
|
1884
2030
|
/** Embed one sub-batch's texts, upsert the vectors, and stamp each surviving chunk row Complete. */
|
|
1885
|
-
async embedAndPersistChunkBatch(embeddable, infra, tagMap, contextUser, stats) {
|
|
2031
|
+
async embedAndPersistChunkBatch(embeddable, infra, tagMap, contextUser, stats, directivesByItem) {
|
|
1886
2032
|
const texts = embeddable.map(e => e.chunk.text);
|
|
1887
2033
|
await this.EmbeddingRateLimiter.Acquire(texts.reduce((sum, t) => sum + Math.ceil(t.length / 4), 0));
|
|
1888
2034
|
const runResult = await new AIModelRunner().RunEmbedding({
|
|
@@ -1906,7 +2052,9 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
|
|
|
1906
2052
|
values: runResult.Vectors[idx],
|
|
1907
2053
|
metadata: this.buildVectorMetadata(e.chunk, false, tagMap.get(e.chunk.item.ID), config, contentItemEntity),
|
|
1908
2054
|
};
|
|
1909
|
-
const directives =
|
|
2055
|
+
const directives = directivesByItem
|
|
2056
|
+
? directivesByItem.get(NormalizeUUID(e.chunk.item.ID))
|
|
2057
|
+
: this.buildProviderDirectives(e.chunk.item, infra);
|
|
1910
2058
|
if (directives) {
|
|
1911
2059
|
record.providerTemporaryDirectives = directives;
|
|
1912
2060
|
}
|