@topgunbuild/core 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +756 -141
- package/dist/index.d.ts +756 -141
- package/dist/index.js +1094 -9
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1094 -9
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -4330,6 +4330,593 @@ _InvertedIndex.RETRIEVAL_COST = 50;
|
|
|
4330
4330
|
_InvertedIndex.SUPPORTED_QUERIES = ["contains", "containsAll", "containsAny", "has"];
|
|
4331
4331
|
var InvertedIndex = _InvertedIndex;
|
|
4332
4332
|
|
|
4333
|
+
// src/query/indexes/CompoundIndex.ts
|
|
4334
|
+
var _CompoundIndex = class _CompoundIndex {
|
|
4335
|
+
/**
|
|
4336
|
+
* Create a CompoundIndex.
|
|
4337
|
+
*
|
|
4338
|
+
* @param attributes - Array of attributes to index (order matters!)
|
|
4339
|
+
* @param options - Optional configuration
|
|
4340
|
+
*
|
|
4341
|
+
* @example
|
|
4342
|
+
* ```typescript
|
|
4343
|
+
* const statusAttr = simpleAttribute<Product, string>('status', p => p.status);
|
|
4344
|
+
* const categoryAttr = simpleAttribute<Product, string>('category', p => p.category);
|
|
4345
|
+
*
|
|
4346
|
+
* const compoundIndex = new CompoundIndex<string, Product>([statusAttr, categoryAttr]);
|
|
4347
|
+
* ```
|
|
4348
|
+
*/
|
|
4349
|
+
constructor(attributes, options = {}) {
|
|
4350
|
+
this.type = "compound";
|
|
4351
|
+
/** Map from composite key to set of record keys */
|
|
4352
|
+
this.data = /* @__PURE__ */ new Map();
|
|
4353
|
+
/** Set of all indexed keys */
|
|
4354
|
+
this.allKeys = /* @__PURE__ */ new Set();
|
|
4355
|
+
if (attributes.length < 2) {
|
|
4356
|
+
throw new Error("CompoundIndex requires at least 2 attributes");
|
|
4357
|
+
}
|
|
4358
|
+
this._attributes = attributes;
|
|
4359
|
+
this.separator = options.separator ?? "|";
|
|
4360
|
+
}
|
|
4361
|
+
/**
|
|
4362
|
+
* Get the first attribute (used for Index interface compatibility).
|
|
4363
|
+
* Note: CompoundIndex spans multiple attributes.
|
|
4364
|
+
*/
|
|
4365
|
+
get attribute() {
|
|
4366
|
+
return this._attributes[0];
|
|
4367
|
+
}
|
|
4368
|
+
/**
|
|
4369
|
+
* Get all attributes in this compound index.
|
|
4370
|
+
*/
|
|
4371
|
+
get attributes() {
|
|
4372
|
+
return [...this._attributes];
|
|
4373
|
+
}
|
|
4374
|
+
/**
|
|
4375
|
+
* Get attribute names as a combined identifier.
|
|
4376
|
+
*/
|
|
4377
|
+
get compoundName() {
|
|
4378
|
+
return this._attributes.map((a) => a.name).join("+");
|
|
4379
|
+
}
|
|
4380
|
+
getRetrievalCost() {
|
|
4381
|
+
return _CompoundIndex.RETRIEVAL_COST;
|
|
4382
|
+
}
|
|
4383
|
+
supportsQuery(queryType) {
|
|
4384
|
+
return queryType === "compound";
|
|
4385
|
+
}
|
|
4386
|
+
/**
|
|
4387
|
+
* Retrieve records matching compound query.
|
|
4388
|
+
*
|
|
4389
|
+
* @param query - Compound query with values matching each attribute
|
|
4390
|
+
* @returns ResultSet of matching keys
|
|
4391
|
+
*
|
|
4392
|
+
* @example
|
|
4393
|
+
* ```typescript
|
|
4394
|
+
* // Find products where status='active' AND category='electronics'
|
|
4395
|
+
* index.retrieve({
|
|
4396
|
+
* type: 'compound',
|
|
4397
|
+
* values: ['active', 'electronics']
|
|
4398
|
+
* });
|
|
4399
|
+
* ```
|
|
4400
|
+
*/
|
|
4401
|
+
retrieve(query) {
|
|
4402
|
+
if (query.type !== "compound") {
|
|
4403
|
+
throw new Error(`CompoundIndex only supports 'compound' query type, got: ${query.type}`);
|
|
4404
|
+
}
|
|
4405
|
+
const compoundQuery = query;
|
|
4406
|
+
const values = compoundQuery.values;
|
|
4407
|
+
if (values.length !== this._attributes.length) {
|
|
4408
|
+
throw new Error(
|
|
4409
|
+
`CompoundIndex requires ${this._attributes.length} values, got ${values.length}`
|
|
4410
|
+
);
|
|
4411
|
+
}
|
|
4412
|
+
const compositeKey = this.buildCompositeKey(values);
|
|
4413
|
+
const keys = this.data.get(compositeKey);
|
|
4414
|
+
return new SetResultSet(
|
|
4415
|
+
keys ? new Set(keys) : /* @__PURE__ */ new Set(),
|
|
4416
|
+
_CompoundIndex.RETRIEVAL_COST
|
|
4417
|
+
);
|
|
4418
|
+
}
|
|
4419
|
+
/**
|
|
4420
|
+
* Retrieve with explicit values (convenience method).
|
|
4421
|
+
*
|
|
4422
|
+
* @param values - Values in order of index attributes
|
|
4423
|
+
* @returns ResultSet of matching keys
|
|
4424
|
+
*/
|
|
4425
|
+
retrieveByValues(...values) {
|
|
4426
|
+
return this.retrieve({ type: "compound", values });
|
|
4427
|
+
}
|
|
4428
|
+
add(key, record) {
|
|
4429
|
+
const compositeKey = this.buildCompositeKeyFromRecord(record);
|
|
4430
|
+
if (compositeKey === null) return;
|
|
4431
|
+
let keys = this.data.get(compositeKey);
|
|
4432
|
+
if (!keys) {
|
|
4433
|
+
keys = /* @__PURE__ */ new Set();
|
|
4434
|
+
this.data.set(compositeKey, keys);
|
|
4435
|
+
}
|
|
4436
|
+
keys.add(key);
|
|
4437
|
+
this.allKeys.add(key);
|
|
4438
|
+
}
|
|
4439
|
+
remove(key, record) {
|
|
4440
|
+
const compositeKey = this.buildCompositeKeyFromRecord(record);
|
|
4441
|
+
if (compositeKey === null) return;
|
|
4442
|
+
const keys = this.data.get(compositeKey);
|
|
4443
|
+
if (keys) {
|
|
4444
|
+
keys.delete(key);
|
|
4445
|
+
if (keys.size === 0) {
|
|
4446
|
+
this.data.delete(compositeKey);
|
|
4447
|
+
}
|
|
4448
|
+
}
|
|
4449
|
+
this.allKeys.delete(key);
|
|
4450
|
+
}
|
|
4451
|
+
update(key, oldRecord, newRecord) {
|
|
4452
|
+
const oldKey = this.buildCompositeKeyFromRecord(oldRecord);
|
|
4453
|
+
const newKey = this.buildCompositeKeyFromRecord(newRecord);
|
|
4454
|
+
if (oldKey === newKey) {
|
|
4455
|
+
return;
|
|
4456
|
+
}
|
|
4457
|
+
this.remove(key, oldRecord);
|
|
4458
|
+
this.add(key, newRecord);
|
|
4459
|
+
}
|
|
4460
|
+
clear() {
|
|
4461
|
+
this.data.clear();
|
|
4462
|
+
this.allKeys.clear();
|
|
4463
|
+
}
|
|
4464
|
+
getStats() {
|
|
4465
|
+
let totalEntries = 0;
|
|
4466
|
+
for (const keys of this.data.values()) {
|
|
4467
|
+
totalEntries += keys.size;
|
|
4468
|
+
}
|
|
4469
|
+
return {
|
|
4470
|
+
distinctValues: this.data.size,
|
|
4471
|
+
totalEntries,
|
|
4472
|
+
avgEntriesPerValue: this.data.size > 0 ? totalEntries / this.data.size : 0
|
|
4473
|
+
};
|
|
4474
|
+
}
|
|
4475
|
+
/**
|
|
4476
|
+
* Get extended statistics for compound index.
|
|
4477
|
+
*/
|
|
4478
|
+
getExtendedStats() {
|
|
4479
|
+
const stats = this.getStats();
|
|
4480
|
+
return {
|
|
4481
|
+
...stats,
|
|
4482
|
+
attributeCount: this._attributes.length,
|
|
4483
|
+
attributeNames: this._attributes.map((a) => a.name),
|
|
4484
|
+
compositeKeyCount: this.data.size
|
|
4485
|
+
};
|
|
4486
|
+
}
|
|
4487
|
+
/**
|
|
4488
|
+
* Check if this compound index can answer a query on the given attributes.
|
|
4489
|
+
* Compound indexes can be used if query attributes match in prefix order.
|
|
4490
|
+
*
|
|
4491
|
+
* @param attributeNames - Attribute names being queried
|
|
4492
|
+
* @returns true if this index can answer the query
|
|
4493
|
+
*/
|
|
4494
|
+
canAnswerQuery(attributeNames) {
|
|
4495
|
+
if (attributeNames.length !== this._attributes.length) {
|
|
4496
|
+
return false;
|
|
4497
|
+
}
|
|
4498
|
+
for (let i = 0; i < attributeNames.length; i++) {
|
|
4499
|
+
if (attributeNames[i] !== this._attributes[i].name) {
|
|
4500
|
+
return false;
|
|
4501
|
+
}
|
|
4502
|
+
}
|
|
4503
|
+
return true;
|
|
4504
|
+
}
|
|
4505
|
+
/**
|
|
4506
|
+
* Build composite key from array of values.
|
|
4507
|
+
*/
|
|
4508
|
+
buildCompositeKey(values) {
|
|
4509
|
+
return values.map((v) => this.encodeValue(v)).join(this.separator);
|
|
4510
|
+
}
|
|
4511
|
+
/**
|
|
4512
|
+
* Build composite key from record by extracting attribute values.
|
|
4513
|
+
* Returns null if any attribute value is undefined.
|
|
4514
|
+
*/
|
|
4515
|
+
buildCompositeKeyFromRecord(record) {
|
|
4516
|
+
const values = [];
|
|
4517
|
+
for (const attr of this._attributes) {
|
|
4518
|
+
const value = attr.getValue(record);
|
|
4519
|
+
if (value === void 0) {
|
|
4520
|
+
return null;
|
|
4521
|
+
}
|
|
4522
|
+
values.push(value);
|
|
4523
|
+
}
|
|
4524
|
+
return this.buildCompositeKey(values);
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Encode value for composite key.
|
|
4528
|
+
* Handles common types and escapes separator.
|
|
4529
|
+
*/
|
|
4530
|
+
encodeValue(value) {
|
|
4531
|
+
if (value === null) return "__null__";
|
|
4532
|
+
if (value === void 0) return "__undefined__";
|
|
4533
|
+
const str = String(value);
|
|
4534
|
+
return str.replace(
|
|
4535
|
+
new RegExp(this.escapeRegex(this.separator), "g"),
|
|
4536
|
+
`\\${this.separator}`
|
|
4537
|
+
);
|
|
4538
|
+
}
|
|
4539
|
+
/**
|
|
4540
|
+
* Escape regex special characters.
|
|
4541
|
+
*/
|
|
4542
|
+
escapeRegex(str) {
|
|
4543
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4544
|
+
}
|
|
4545
|
+
};
|
|
4546
|
+
/** Retrieval cost (lower than individual indexes combined) */
|
|
4547
|
+
_CompoundIndex.RETRIEVAL_COST = 20;
|
|
4548
|
+
var CompoundIndex = _CompoundIndex;
|
|
4549
|
+
function isCompoundIndex(index) {
|
|
4550
|
+
return index.type === "compound";
|
|
4551
|
+
}
|
|
4552
|
+
|
|
4553
|
+
// src/query/indexes/lazy/LazyHashIndex.ts
|
|
4554
|
+
var LazyHashIndex = class {
|
|
4555
|
+
constructor(attribute, options = {}) {
|
|
4556
|
+
this.attribute = attribute;
|
|
4557
|
+
this.type = "hash";
|
|
4558
|
+
this.isLazy = true;
|
|
4559
|
+
/** Underlying hash index (created on first query) */
|
|
4560
|
+
this.innerIndex = null;
|
|
4561
|
+
/** Pending records before materialization */
|
|
4562
|
+
this.pendingRecords = /* @__PURE__ */ new Map();
|
|
4563
|
+
/** Track if index has been built */
|
|
4564
|
+
this.built = false;
|
|
4565
|
+
this.onProgress = options.onProgress;
|
|
4566
|
+
this.progressBatchSize = options.progressBatchSize ?? 1e3;
|
|
4567
|
+
}
|
|
4568
|
+
get isBuilt() {
|
|
4569
|
+
return this.built;
|
|
4570
|
+
}
|
|
4571
|
+
get pendingCount() {
|
|
4572
|
+
return this.pendingRecords.size;
|
|
4573
|
+
}
|
|
4574
|
+
getRetrievalCost() {
|
|
4575
|
+
return 30;
|
|
4576
|
+
}
|
|
4577
|
+
supportsQuery(queryType) {
|
|
4578
|
+
return ["equal", "in", "has"].includes(queryType);
|
|
4579
|
+
}
|
|
4580
|
+
retrieve(query) {
|
|
4581
|
+
if (!this.built) {
|
|
4582
|
+
this.materialize();
|
|
4583
|
+
}
|
|
4584
|
+
return this.innerIndex.retrieve(query);
|
|
4585
|
+
}
|
|
4586
|
+
add(key, record) {
|
|
4587
|
+
if (this.built) {
|
|
4588
|
+
this.innerIndex.add(key, record);
|
|
4589
|
+
} else {
|
|
4590
|
+
this.pendingRecords.set(key, record);
|
|
4591
|
+
}
|
|
4592
|
+
}
|
|
4593
|
+
remove(key, record) {
|
|
4594
|
+
if (this.built) {
|
|
4595
|
+
this.innerIndex.remove(key, record);
|
|
4596
|
+
} else {
|
|
4597
|
+
this.pendingRecords.delete(key);
|
|
4598
|
+
}
|
|
4599
|
+
}
|
|
4600
|
+
update(key, oldRecord, newRecord) {
|
|
4601
|
+
if (this.built) {
|
|
4602
|
+
this.innerIndex.update(key, oldRecord, newRecord);
|
|
4603
|
+
} else {
|
|
4604
|
+
this.pendingRecords.set(key, newRecord);
|
|
4605
|
+
}
|
|
4606
|
+
}
|
|
4607
|
+
clear() {
|
|
4608
|
+
if (this.built) {
|
|
4609
|
+
this.innerIndex.clear();
|
|
4610
|
+
}
|
|
4611
|
+
this.pendingRecords.clear();
|
|
4612
|
+
}
|
|
4613
|
+
getStats() {
|
|
4614
|
+
if (this.built) {
|
|
4615
|
+
return this.innerIndex.getStats();
|
|
4616
|
+
}
|
|
4617
|
+
return {
|
|
4618
|
+
distinctValues: 0,
|
|
4619
|
+
totalEntries: this.pendingRecords.size,
|
|
4620
|
+
avgEntriesPerValue: 0
|
|
4621
|
+
};
|
|
4622
|
+
}
|
|
4623
|
+
/**
|
|
4624
|
+
* Force materialization of the index.
|
|
4625
|
+
* Called automatically on first query.
|
|
4626
|
+
*/
|
|
4627
|
+
materialize(progressCallback) {
|
|
4628
|
+
if (this.built) return;
|
|
4629
|
+
const callback = progressCallback ?? this.onProgress;
|
|
4630
|
+
const total = this.pendingRecords.size;
|
|
4631
|
+
this.innerIndex = new HashIndex(this.attribute);
|
|
4632
|
+
let processed = 0;
|
|
4633
|
+
for (const [key, record] of this.pendingRecords) {
|
|
4634
|
+
this.innerIndex.add(key, record);
|
|
4635
|
+
processed++;
|
|
4636
|
+
if (callback && processed % this.progressBatchSize === 0) {
|
|
4637
|
+
const progress = Math.round(processed / total * 100);
|
|
4638
|
+
callback(this.attribute.name, progress, processed, total);
|
|
4639
|
+
}
|
|
4640
|
+
}
|
|
4641
|
+
if (callback && total > 0) {
|
|
4642
|
+
callback(this.attribute.name, 100, total, total);
|
|
4643
|
+
}
|
|
4644
|
+
this.pendingRecords.clear();
|
|
4645
|
+
this.built = true;
|
|
4646
|
+
}
|
|
4647
|
+
/**
|
|
4648
|
+
* Get the underlying HashIndex (for testing/debugging).
|
|
4649
|
+
* Returns null if not yet materialized.
|
|
4650
|
+
*/
|
|
4651
|
+
getInnerIndex() {
|
|
4652
|
+
return this.innerIndex;
|
|
4653
|
+
}
|
|
4654
|
+
};
|
|
4655
|
+
|
|
4656
|
+
// src/query/indexes/lazy/LazyNavigableIndex.ts
|
|
4657
|
+
var LazyNavigableIndex = class {
|
|
4658
|
+
constructor(attribute, comparator, options = {}) {
|
|
4659
|
+
this.attribute = attribute;
|
|
4660
|
+
this.type = "navigable";
|
|
4661
|
+
this.isLazy = true;
|
|
4662
|
+
/** Underlying navigable index (created on first query) */
|
|
4663
|
+
this.innerIndex = null;
|
|
4664
|
+
/** Pending records before materialization */
|
|
4665
|
+
this.pendingRecords = /* @__PURE__ */ new Map();
|
|
4666
|
+
/** Track if index has been built */
|
|
4667
|
+
this.built = false;
|
|
4668
|
+
this.comparator = comparator;
|
|
4669
|
+
this.onProgress = options.onProgress;
|
|
4670
|
+
this.progressBatchSize = options.progressBatchSize ?? 1e3;
|
|
4671
|
+
}
|
|
4672
|
+
get isBuilt() {
|
|
4673
|
+
return this.built;
|
|
4674
|
+
}
|
|
4675
|
+
get pendingCount() {
|
|
4676
|
+
return this.pendingRecords.size;
|
|
4677
|
+
}
|
|
4678
|
+
getRetrievalCost() {
|
|
4679
|
+
return 40;
|
|
4680
|
+
}
|
|
4681
|
+
supportsQuery(queryType) {
|
|
4682
|
+
return ["equal", "in", "has", "gt", "gte", "lt", "lte", "between"].includes(queryType);
|
|
4683
|
+
}
|
|
4684
|
+
retrieve(query) {
|
|
4685
|
+
if (!this.built) {
|
|
4686
|
+
this.materialize();
|
|
4687
|
+
}
|
|
4688
|
+
return this.innerIndex.retrieve(query);
|
|
4689
|
+
}
|
|
4690
|
+
add(key, record) {
|
|
4691
|
+
if (this.built) {
|
|
4692
|
+
this.innerIndex.add(key, record);
|
|
4693
|
+
} else {
|
|
4694
|
+
this.pendingRecords.set(key, record);
|
|
4695
|
+
}
|
|
4696
|
+
}
|
|
4697
|
+
remove(key, record) {
|
|
4698
|
+
if (this.built) {
|
|
4699
|
+
this.innerIndex.remove(key, record);
|
|
4700
|
+
} else {
|
|
4701
|
+
this.pendingRecords.delete(key);
|
|
4702
|
+
}
|
|
4703
|
+
}
|
|
4704
|
+
update(key, oldRecord, newRecord) {
|
|
4705
|
+
if (this.built) {
|
|
4706
|
+
this.innerIndex.update(key, oldRecord, newRecord);
|
|
4707
|
+
} else {
|
|
4708
|
+
this.pendingRecords.set(key, newRecord);
|
|
4709
|
+
}
|
|
4710
|
+
}
|
|
4711
|
+
clear() {
|
|
4712
|
+
if (this.built) {
|
|
4713
|
+
this.innerIndex.clear();
|
|
4714
|
+
}
|
|
4715
|
+
this.pendingRecords.clear();
|
|
4716
|
+
}
|
|
4717
|
+
getStats() {
|
|
4718
|
+
if (this.built) {
|
|
4719
|
+
return this.innerIndex.getStats();
|
|
4720
|
+
}
|
|
4721
|
+
return {
|
|
4722
|
+
distinctValues: 0,
|
|
4723
|
+
totalEntries: this.pendingRecords.size,
|
|
4724
|
+
avgEntriesPerValue: 0
|
|
4725
|
+
};
|
|
4726
|
+
}
|
|
4727
|
+
/**
|
|
4728
|
+
* Force materialization of the index.
|
|
4729
|
+
* Called automatically on first query.
|
|
4730
|
+
*/
|
|
4731
|
+
materialize(progressCallback) {
|
|
4732
|
+
if (this.built) return;
|
|
4733
|
+
const callback = progressCallback ?? this.onProgress;
|
|
4734
|
+
const total = this.pendingRecords.size;
|
|
4735
|
+
this.innerIndex = new NavigableIndex(this.attribute, this.comparator);
|
|
4736
|
+
let processed = 0;
|
|
4737
|
+
for (const [key, record] of this.pendingRecords) {
|
|
4738
|
+
this.innerIndex.add(key, record);
|
|
4739
|
+
processed++;
|
|
4740
|
+
if (callback && processed % this.progressBatchSize === 0) {
|
|
4741
|
+
const progress = Math.round(processed / total * 100);
|
|
4742
|
+
callback(this.attribute.name, progress, processed, total);
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
if (callback && total > 0) {
|
|
4746
|
+
callback(this.attribute.name, 100, total, total);
|
|
4747
|
+
}
|
|
4748
|
+
this.pendingRecords.clear();
|
|
4749
|
+
this.built = true;
|
|
4750
|
+
}
|
|
4751
|
+
/**
|
|
4752
|
+
* Get the underlying NavigableIndex (for testing/debugging).
|
|
4753
|
+
* Returns null if not yet materialized.
|
|
4754
|
+
*/
|
|
4755
|
+
getInnerIndex() {
|
|
4756
|
+
return this.innerIndex;
|
|
4757
|
+
}
|
|
4758
|
+
/**
|
|
4759
|
+
* Get the minimum indexed value.
|
|
4760
|
+
* Forces materialization if not built.
|
|
4761
|
+
*/
|
|
4762
|
+
getMinValue() {
|
|
4763
|
+
if (!this.built) {
|
|
4764
|
+
this.materialize();
|
|
4765
|
+
}
|
|
4766
|
+
return this.innerIndex.getMinValue();
|
|
4767
|
+
}
|
|
4768
|
+
/**
|
|
4769
|
+
* Get the maximum indexed value.
|
|
4770
|
+
* Forces materialization if not built.
|
|
4771
|
+
*/
|
|
4772
|
+
getMaxValue() {
|
|
4773
|
+
if (!this.built) {
|
|
4774
|
+
this.materialize();
|
|
4775
|
+
}
|
|
4776
|
+
return this.innerIndex.getMaxValue();
|
|
4777
|
+
}
|
|
4778
|
+
};
|
|
4779
|
+
|
|
4780
|
+
// src/query/indexes/lazy/LazyInvertedIndex.ts
|
|
4781
|
+
var LazyInvertedIndex = class {
|
|
4782
|
+
constructor(attribute, pipeline, options = {}) {
|
|
4783
|
+
this.attribute = attribute;
|
|
4784
|
+
this.type = "inverted";
|
|
4785
|
+
this.isLazy = true;
|
|
4786
|
+
/** Underlying inverted index (created on first query) */
|
|
4787
|
+
this.innerIndex = null;
|
|
4788
|
+
/** Pending records before materialization */
|
|
4789
|
+
this.pendingRecords = /* @__PURE__ */ new Map();
|
|
4790
|
+
/** Track if index has been built */
|
|
4791
|
+
this.built = false;
|
|
4792
|
+
this.pipeline = pipeline ?? TokenizationPipeline.simple();
|
|
4793
|
+
this.onProgress = options.onProgress;
|
|
4794
|
+
this.progressBatchSize = options.progressBatchSize ?? 1e3;
|
|
4795
|
+
}
|
|
4796
|
+
get isBuilt() {
|
|
4797
|
+
return this.built;
|
|
4798
|
+
}
|
|
4799
|
+
get pendingCount() {
|
|
4800
|
+
return this.pendingRecords.size;
|
|
4801
|
+
}
|
|
4802
|
+
getRetrievalCost() {
|
|
4803
|
+
return 50;
|
|
4804
|
+
}
|
|
4805
|
+
supportsQuery(queryType) {
|
|
4806
|
+
return ["contains", "containsAll", "containsAny", "has"].includes(queryType);
|
|
4807
|
+
}
|
|
4808
|
+
retrieve(query) {
|
|
4809
|
+
if (!this.built) {
|
|
4810
|
+
this.materialize();
|
|
4811
|
+
}
|
|
4812
|
+
return this.innerIndex.retrieve(query);
|
|
4813
|
+
}
|
|
4814
|
+
add(key, record) {
|
|
4815
|
+
if (this.built) {
|
|
4816
|
+
this.innerIndex.add(key, record);
|
|
4817
|
+
} else {
|
|
4818
|
+
this.pendingRecords.set(key, record);
|
|
4819
|
+
}
|
|
4820
|
+
}
|
|
4821
|
+
remove(key, record) {
|
|
4822
|
+
if (this.built) {
|
|
4823
|
+
this.innerIndex.remove(key, record);
|
|
4824
|
+
} else {
|
|
4825
|
+
this.pendingRecords.delete(key);
|
|
4826
|
+
}
|
|
4827
|
+
}
|
|
4828
|
+
update(key, oldRecord, newRecord) {
|
|
4829
|
+
if (this.built) {
|
|
4830
|
+
this.innerIndex.update(key, oldRecord, newRecord);
|
|
4831
|
+
} else {
|
|
4832
|
+
this.pendingRecords.set(key, newRecord);
|
|
4833
|
+
}
|
|
4834
|
+
}
|
|
4835
|
+
clear() {
|
|
4836
|
+
if (this.built) {
|
|
4837
|
+
this.innerIndex.clear();
|
|
4838
|
+
}
|
|
4839
|
+
this.pendingRecords.clear();
|
|
4840
|
+
}
|
|
4841
|
+
getStats() {
|
|
4842
|
+
if (this.built) {
|
|
4843
|
+
return this.innerIndex.getStats();
|
|
4844
|
+
}
|
|
4845
|
+
return {
|
|
4846
|
+
distinctValues: 0,
|
|
4847
|
+
totalEntries: this.pendingRecords.size,
|
|
4848
|
+
avgEntriesPerValue: 0
|
|
4849
|
+
};
|
|
4850
|
+
}
|
|
4851
|
+
/**
|
|
4852
|
+
* Get extended statistics for full-text index.
|
|
4853
|
+
* Forces materialization if not built.
|
|
4854
|
+
*/
|
|
4855
|
+
getExtendedStats() {
|
|
4856
|
+
if (!this.built) {
|
|
4857
|
+
this.materialize();
|
|
4858
|
+
}
|
|
4859
|
+
return this.innerIndex.getExtendedStats();
|
|
4860
|
+
}
|
|
4861
|
+
/**
|
|
4862
|
+
* Force materialization of the index.
|
|
4863
|
+
* Called automatically on first query.
|
|
4864
|
+
*/
|
|
4865
|
+
materialize(progressCallback) {
|
|
4866
|
+
if (this.built) return;
|
|
4867
|
+
const callback = progressCallback ?? this.onProgress;
|
|
4868
|
+
const total = this.pendingRecords.size;
|
|
4869
|
+
this.innerIndex = new InvertedIndex(this.attribute, this.pipeline);
|
|
4870
|
+
let processed = 0;
|
|
4871
|
+
for (const [key, record] of this.pendingRecords) {
|
|
4872
|
+
this.innerIndex.add(key, record);
|
|
4873
|
+
processed++;
|
|
4874
|
+
if (callback && processed % this.progressBatchSize === 0) {
|
|
4875
|
+
const progress = Math.round(processed / total * 100);
|
|
4876
|
+
callback(this.attribute.name, progress, processed, total);
|
|
4877
|
+
}
|
|
4878
|
+
}
|
|
4879
|
+
if (callback && total > 0) {
|
|
4880
|
+
callback(this.attribute.name, 100, total, total);
|
|
4881
|
+
}
|
|
4882
|
+
this.pendingRecords.clear();
|
|
4883
|
+
this.built = true;
|
|
4884
|
+
}
|
|
4885
|
+
/**
|
|
4886
|
+
* Get the underlying InvertedIndex (for testing/debugging).
|
|
4887
|
+
* Returns null if not yet materialized.
|
|
4888
|
+
*/
|
|
4889
|
+
getInnerIndex() {
|
|
4890
|
+
return this.innerIndex;
|
|
4891
|
+
}
|
|
4892
|
+
/**
|
|
4893
|
+
* Get the tokenization pipeline.
|
|
4894
|
+
*/
|
|
4895
|
+
getPipeline() {
|
|
4896
|
+
return this.pipeline;
|
|
4897
|
+
}
|
|
4898
|
+
/**
|
|
4899
|
+
* Check if a specific token exists in the index.
|
|
4900
|
+
* Forces materialization if not built.
|
|
4901
|
+
*/
|
|
4902
|
+
hasToken(token) {
|
|
4903
|
+
if (!this.built) {
|
|
4904
|
+
this.materialize();
|
|
4905
|
+
}
|
|
4906
|
+
return this.innerIndex.hasToken(token);
|
|
4907
|
+
}
|
|
4908
|
+
/**
|
|
4909
|
+
* Get the number of documents for a specific token.
|
|
4910
|
+
* Forces materialization if not built.
|
|
4911
|
+
*/
|
|
4912
|
+
getTokenDocumentCount(token) {
|
|
4913
|
+
if (!this.built) {
|
|
4914
|
+
this.materialize();
|
|
4915
|
+
}
|
|
4916
|
+
return this.innerIndex.getTokenDocumentCount(token);
|
|
4917
|
+
}
|
|
4918
|
+
};
|
|
4919
|
+
|
|
4333
4920
|
// src/query/resultset/IntersectionResultSet.ts
|
|
4334
4921
|
var IntersectionResultSet = class {
|
|
4335
4922
|
/**
|
|
@@ -4883,6 +5470,8 @@ var IndexRegistry = class {
|
|
|
4883
5470
|
constructor() {
|
|
4884
5471
|
/** Indexes grouped by attribute name */
|
|
4885
5472
|
this.attributeIndexes = /* @__PURE__ */ new Map();
|
|
5473
|
+
/** Compound indexes (Phase 9.03) - keyed by sorted attribute names */
|
|
5474
|
+
this.compoundIndexes = /* @__PURE__ */ new Map();
|
|
4886
5475
|
/** Fallback index for full scan (optional) */
|
|
4887
5476
|
this.fallbackIndex = null;
|
|
4888
5477
|
}
|
|
@@ -4893,6 +5482,10 @@ var IndexRegistry = class {
|
|
|
4893
5482
|
* @param index - Index to register
|
|
4894
5483
|
*/
|
|
4895
5484
|
addIndex(index) {
|
|
5485
|
+
if (isCompoundIndex(index)) {
|
|
5486
|
+
this.addCompoundIndex(index);
|
|
5487
|
+
return;
|
|
5488
|
+
}
|
|
4896
5489
|
const attrName = index.attribute.name;
|
|
4897
5490
|
let indexes = this.attributeIndexes.get(attrName);
|
|
4898
5491
|
if (!indexes) {
|
|
@@ -4903,6 +5496,15 @@ var IndexRegistry = class {
|
|
|
4903
5496
|
indexes.push(index);
|
|
4904
5497
|
}
|
|
4905
5498
|
}
|
|
5499
|
+
/**
|
|
5500
|
+
* Register a compound index (Phase 9.03).
|
|
5501
|
+
*
|
|
5502
|
+
* @param index - Compound index to register
|
|
5503
|
+
*/
|
|
5504
|
+
addCompoundIndex(index) {
|
|
5505
|
+
const key = this.makeCompoundKey(index.attributes.map((a) => a.name));
|
|
5506
|
+
this.compoundIndexes.set(key, index);
|
|
5507
|
+
}
|
|
4906
5508
|
/**
|
|
4907
5509
|
* Remove an index from the registry.
|
|
4908
5510
|
*
|
|
@@ -4910,6 +5512,9 @@ var IndexRegistry = class {
|
|
|
4910
5512
|
* @returns true if index was found and removed
|
|
4911
5513
|
*/
|
|
4912
5514
|
removeIndex(index) {
|
|
5515
|
+
if (isCompoundIndex(index)) {
|
|
5516
|
+
return this.removeCompoundIndex(index);
|
|
5517
|
+
}
|
|
4913
5518
|
const attrName = index.attribute.name;
|
|
4914
5519
|
const indexes = this.attributeIndexes.get(attrName);
|
|
4915
5520
|
if (!indexes) {
|
|
@@ -4925,6 +5530,16 @@ var IndexRegistry = class {
|
|
|
4925
5530
|
}
|
|
4926
5531
|
return true;
|
|
4927
5532
|
}
|
|
5533
|
+
/**
|
|
5534
|
+
* Remove a compound index (Phase 9.03).
|
|
5535
|
+
*
|
|
5536
|
+
* @param index - Compound index to remove
|
|
5537
|
+
* @returns true if index was found and removed
|
|
5538
|
+
*/
|
|
5539
|
+
removeCompoundIndex(index) {
|
|
5540
|
+
const key = this.makeCompoundKey(index.attributes.map((a) => a.name));
|
|
5541
|
+
return this.compoundIndexes.delete(key);
|
|
5542
|
+
}
|
|
4928
5543
|
/**
|
|
4929
5544
|
* Get all indexes for an attribute.
|
|
4930
5545
|
*
|
|
@@ -4995,6 +5610,50 @@ var IndexRegistry = class {
|
|
|
4995
5610
|
const indexes = this.getIndexes(attributeName);
|
|
4996
5611
|
return indexes.filter((index) => index.supportsQuery(queryType)).sort((a, b) => a.getRetrievalCost() - b.getRetrievalCost());
|
|
4997
5612
|
}
|
|
5613
|
+
// ========================================
|
|
5614
|
+
// Phase 9.03: Compound Index Methods
|
|
5615
|
+
// ========================================
|
|
5616
|
+
/**
|
|
5617
|
+
* Find a compound index that covers the given attribute names (Phase 9.03).
|
|
5618
|
+
* The compound index must cover ALL the attributes (exact match or superset).
|
|
5619
|
+
*
|
|
5620
|
+
* @param attributeNames - Array of attribute names to search for
|
|
5621
|
+
* @returns Matching compound index or null
|
|
5622
|
+
*/
|
|
5623
|
+
findCompoundIndex(attributeNames) {
|
|
5624
|
+
if (attributeNames.length < 2) {
|
|
5625
|
+
return null;
|
|
5626
|
+
}
|
|
5627
|
+
const key = this.makeCompoundKey(attributeNames);
|
|
5628
|
+
const exactMatch = this.compoundIndexes.get(key);
|
|
5629
|
+
if (exactMatch) {
|
|
5630
|
+
return exactMatch;
|
|
5631
|
+
}
|
|
5632
|
+
return null;
|
|
5633
|
+
}
|
|
5634
|
+
/**
|
|
5635
|
+
* Check if a compound index exists for the given attributes (Phase 9.03).
|
|
5636
|
+
*
|
|
5637
|
+
* @param attributeNames - Array of attribute names
|
|
5638
|
+
* @returns true if a compound index exists
|
|
5639
|
+
*/
|
|
5640
|
+
hasCompoundIndex(attributeNames) {
|
|
5641
|
+
return this.findCompoundIndex(attributeNames) !== null;
|
|
5642
|
+
}
|
|
5643
|
+
/**
|
|
5644
|
+
* Get all compound indexes (Phase 9.03).
|
|
5645
|
+
*
|
|
5646
|
+
* @returns Array of all compound indexes
|
|
5647
|
+
*/
|
|
5648
|
+
getCompoundIndexes() {
|
|
5649
|
+
return Array.from(this.compoundIndexes.values());
|
|
5650
|
+
}
|
|
5651
|
+
/**
|
|
5652
|
+
* Create a compound key from attribute names (sorted for consistency).
|
|
5653
|
+
*/
|
|
5654
|
+
makeCompoundKey(attributeNames) {
|
|
5655
|
+
return [...attributeNames].sort().join("+");
|
|
5656
|
+
}
|
|
4998
5657
|
/**
|
|
4999
5658
|
* Set a fallback index for queries without a suitable index.
|
|
5000
5659
|
* Typically a FallbackIndex that performs full scan.
|
|
@@ -5025,6 +5684,9 @@ var IndexRegistry = class {
|
|
|
5025
5684
|
index.add(key, record);
|
|
5026
5685
|
}
|
|
5027
5686
|
}
|
|
5687
|
+
for (const compoundIndex of this.compoundIndexes.values()) {
|
|
5688
|
+
compoundIndex.add(key, record);
|
|
5689
|
+
}
|
|
5028
5690
|
}
|
|
5029
5691
|
/**
|
|
5030
5692
|
* Notify all indexes of a record update.
|
|
@@ -5040,6 +5702,9 @@ var IndexRegistry = class {
|
|
|
5040
5702
|
index.update(key, oldRecord, newRecord);
|
|
5041
5703
|
}
|
|
5042
5704
|
}
|
|
5705
|
+
for (const compoundIndex of this.compoundIndexes.values()) {
|
|
5706
|
+
compoundIndex.update(key, oldRecord, newRecord);
|
|
5707
|
+
}
|
|
5043
5708
|
}
|
|
5044
5709
|
/**
|
|
5045
5710
|
* Notify all indexes of a record removal.
|
|
@@ -5054,6 +5719,9 @@ var IndexRegistry = class {
|
|
|
5054
5719
|
index.remove(key, record);
|
|
5055
5720
|
}
|
|
5056
5721
|
}
|
|
5722
|
+
for (const compoundIndex of this.compoundIndexes.values()) {
|
|
5723
|
+
compoundIndex.remove(key, record);
|
|
5724
|
+
}
|
|
5057
5725
|
}
|
|
5058
5726
|
/**
|
|
5059
5727
|
* Clear all indexes.
|
|
@@ -5065,6 +5733,9 @@ var IndexRegistry = class {
|
|
|
5065
5733
|
index.clear();
|
|
5066
5734
|
}
|
|
5067
5735
|
}
|
|
5736
|
+
for (const compoundIndex of this.compoundIndexes.values()) {
|
|
5737
|
+
compoundIndex.clear();
|
|
5738
|
+
}
|
|
5068
5739
|
}
|
|
5069
5740
|
/**
|
|
5070
5741
|
* Get total number of registered indexes.
|
|
@@ -5074,6 +5745,7 @@ var IndexRegistry = class {
|
|
|
5074
5745
|
for (const indexes of this.attributeIndexes.values()) {
|
|
5075
5746
|
count += indexes.length;
|
|
5076
5747
|
}
|
|
5748
|
+
count += this.compoundIndexes.size;
|
|
5077
5749
|
return count;
|
|
5078
5750
|
}
|
|
5079
5751
|
/**
|
|
@@ -5086,9 +5758,17 @@ var IndexRegistry = class {
|
|
|
5086
5758
|
type: index.type,
|
|
5087
5759
|
stats: index.getStats()
|
|
5088
5760
|
}));
|
|
5761
|
+
for (const compoundIndex of this.compoundIndexes.values()) {
|
|
5762
|
+
indexStats.push({
|
|
5763
|
+
attribute: compoundIndex.compoundName,
|
|
5764
|
+
type: "compound",
|
|
5765
|
+
stats: compoundIndex.getStats()
|
|
5766
|
+
});
|
|
5767
|
+
}
|
|
5089
5768
|
return {
|
|
5090
|
-
totalIndexes: indexes.length,
|
|
5769
|
+
totalIndexes: indexes.length + this.compoundIndexes.size,
|
|
5091
5770
|
indexedAttributes: this.getIndexedAttributes().length,
|
|
5771
|
+
compoundIndexes: this.compoundIndexes.size,
|
|
5092
5772
|
indexes: indexStats
|
|
5093
5773
|
};
|
|
5094
5774
|
}
|
|
@@ -5222,9 +5902,10 @@ var QueryOptimizer = class {
|
|
|
5222
5902
|
* Strategy: Find child with lowest cost, use as base, filter with rest.
|
|
5223
5903
|
*
|
|
5224
5904
|
* CQEngine "smallest first" strategy:
|
|
5225
|
-
* 1.
|
|
5226
|
-
* 2.
|
|
5227
|
-
* 3.
|
|
5905
|
+
* 1. Check for CompoundIndex covering all eq children (Phase 9.03)
|
|
5906
|
+
* 2. Sort children by merge cost
|
|
5907
|
+
* 3. Use intersection if multiple indexes available
|
|
5908
|
+
* 4. Apply remaining predicates as filters
|
|
5228
5909
|
*/
|
|
5229
5910
|
optimizeAnd(query) {
|
|
5230
5911
|
if (!query.children || query.children.length === 0) {
|
|
@@ -5233,6 +5914,10 @@ var QueryOptimizer = class {
|
|
|
5233
5914
|
if (query.children.length === 1) {
|
|
5234
5915
|
return this.optimizeNode(query.children[0]);
|
|
5235
5916
|
}
|
|
5917
|
+
const compoundStep = this.tryCompoundIndex(query.children);
|
|
5918
|
+
if (compoundStep) {
|
|
5919
|
+
return compoundStep;
|
|
5920
|
+
}
|
|
5236
5921
|
const childSteps = query.children.map((child) => this.optimizeNode(child));
|
|
5237
5922
|
const sortedWithIndex = childSteps.map((step, index) => ({ step, originalIndex: index })).sort((a, b) => this.estimateCost(a.step) - this.estimateCost(b.step));
|
|
5238
5923
|
const sortedSteps = sortedWithIndex.map((s) => s.step);
|
|
@@ -5257,6 +5942,71 @@ var QueryOptimizer = class {
|
|
|
5257
5942
|
}
|
|
5258
5943
|
return { type: "intersection", steps: indexedSteps };
|
|
5259
5944
|
}
|
|
5945
|
+
/**
|
|
5946
|
+
* Try to use a CompoundIndex for an AND query (Phase 9.03).
|
|
5947
|
+
*
|
|
5948
|
+
* Returns a compound index scan step if:
|
|
5949
|
+
* 1. All children are simple 'eq' queries
|
|
5950
|
+
* 2. A CompoundIndex exists covering all queried attributes
|
|
5951
|
+
*
|
|
5952
|
+
* @param children - Children of the AND query
|
|
5953
|
+
* @returns IndexScanStep using CompoundIndex, or null if not applicable
|
|
5954
|
+
*/
|
|
5955
|
+
tryCompoundIndex(children) {
|
|
5956
|
+
const eqQueries = [];
|
|
5957
|
+
const otherQueries = [];
|
|
5958
|
+
for (const child of children) {
|
|
5959
|
+
if (isSimpleQuery(child) && child.type === "eq") {
|
|
5960
|
+
eqQueries.push(child);
|
|
5961
|
+
} else {
|
|
5962
|
+
otherQueries.push(child);
|
|
5963
|
+
}
|
|
5964
|
+
}
|
|
5965
|
+
if (eqQueries.length < 2) {
|
|
5966
|
+
return null;
|
|
5967
|
+
}
|
|
5968
|
+
const attributeNames = eqQueries.map((q) => q.attribute);
|
|
5969
|
+
const compoundIndex = this.indexRegistry.findCompoundIndex(attributeNames);
|
|
5970
|
+
if (!compoundIndex) {
|
|
5971
|
+
return null;
|
|
5972
|
+
}
|
|
5973
|
+
const values = this.buildCompoundValues(compoundIndex, eqQueries);
|
|
5974
|
+
if (!values) {
|
|
5975
|
+
return null;
|
|
5976
|
+
}
|
|
5977
|
+
const compoundStep = {
|
|
5978
|
+
type: "index-scan",
|
|
5979
|
+
index: compoundIndex,
|
|
5980
|
+
query: { type: "compound", values }
|
|
5981
|
+
};
|
|
5982
|
+
if (otherQueries.length > 0) {
|
|
5983
|
+
const filterPredicate = otherQueries.length === 1 ? otherQueries[0] : { type: "and", children: otherQueries };
|
|
5984
|
+
return { type: "filter", source: compoundStep, predicate: filterPredicate };
|
|
5985
|
+
}
|
|
5986
|
+
return compoundStep;
|
|
5987
|
+
}
|
|
5988
|
+
/**
|
|
5989
|
+
* Build values array for compound index query in correct attribute order.
|
|
5990
|
+
*
|
|
5991
|
+
* @param compoundIndex - The compound index to use
|
|
5992
|
+
* @param eqQueries - Array of 'eq' queries
|
|
5993
|
+
* @returns Values array in compound index order, or null if mismatch
|
|
5994
|
+
*/
|
|
5995
|
+
buildCompoundValues(compoundIndex, eqQueries) {
|
|
5996
|
+
const attributeNames = compoundIndex.attributes.map((a) => a.name);
|
|
5997
|
+
const values = [];
|
|
5998
|
+
const queryMap = /* @__PURE__ */ new Map();
|
|
5999
|
+
for (const q of eqQueries) {
|
|
6000
|
+
queryMap.set(q.attribute, q.value);
|
|
6001
|
+
}
|
|
6002
|
+
for (const attrName of attributeNames) {
|
|
6003
|
+
if (!queryMap.has(attrName)) {
|
|
6004
|
+
return null;
|
|
6005
|
+
}
|
|
6006
|
+
values.push(queryMap.get(attrName));
|
|
6007
|
+
}
|
|
6008
|
+
return values;
|
|
6009
|
+
}
|
|
5260
6010
|
/**
|
|
5261
6011
|
* Optimize OR query.
|
|
5262
6012
|
* Strategy: Union of all child results with deduplication.
|
|
@@ -5813,7 +6563,9 @@ var MEMORY_OVERHEAD_ESTIMATES = {
|
|
|
5813
6563
|
/** Navigable index overhead per record */
|
|
5814
6564
|
navigable: 32,
|
|
5815
6565
|
/** Inverted index overhead per record (depends on token count) */
|
|
5816
|
-
inverted: 48
|
|
6566
|
+
inverted: 48,
|
|
6567
|
+
/** Compound index overhead per record (includes composite key) */
|
|
6568
|
+
compound: 40
|
|
5817
6569
|
};
|
|
5818
6570
|
|
|
5819
6571
|
// src/query/adaptive/QueryPatternTracker.ts
|
|
@@ -5830,6 +6582,7 @@ function parseStatsKey(key) {
|
|
|
5830
6582
|
var QueryPatternTracker = class {
|
|
5831
6583
|
constructor(options = {}) {
|
|
5832
6584
|
this.stats = /* @__PURE__ */ new Map();
|
|
6585
|
+
this.compoundStats = /* @__PURE__ */ new Map();
|
|
5833
6586
|
this.queryCounter = 0;
|
|
5834
6587
|
this.samplingRate = options.samplingRate ?? TRACKING_SAMPLE_RATE;
|
|
5835
6588
|
this.maxTrackedPatterns = options.maxTrackedPatterns ?? 1e3;
|
|
@@ -5876,6 +6629,94 @@ var QueryPatternTracker = class {
|
|
|
5876
6629
|
});
|
|
5877
6630
|
}
|
|
5878
6631
|
}
|
|
6632
|
+
/**
|
|
6633
|
+
* Record a compound (AND) query execution for pattern tracking (Phase 9.03).
|
|
6634
|
+
*
|
|
6635
|
+
* @param attributes - Array of attribute names being queried together
|
|
6636
|
+
* @param executionTime - Query execution time in milliseconds
|
|
6637
|
+
* @param resultSize - Number of results returned
|
|
6638
|
+
* @param hasCompoundIndex - Whether a compound index was used
|
|
6639
|
+
*/
|
|
6640
|
+
recordCompoundQuery(attributes, executionTime, resultSize, hasCompoundIndex) {
|
|
6641
|
+
if (attributes.length < 2) return;
|
|
6642
|
+
this.queryCounter++;
|
|
6643
|
+
if (this.samplingRate > 1 && this.queryCounter % this.samplingRate !== 0) {
|
|
6644
|
+
return;
|
|
6645
|
+
}
|
|
6646
|
+
const sortedAttrs = [...attributes].sort();
|
|
6647
|
+
const compoundKey = sortedAttrs.join("+");
|
|
6648
|
+
const existing = this.compoundStats.get(compoundKey);
|
|
6649
|
+
const now = Date.now();
|
|
6650
|
+
if (existing) {
|
|
6651
|
+
existing.queryCount++;
|
|
6652
|
+
existing.totalCost += executionTime;
|
|
6653
|
+
existing.averageCost = existing.totalCost / existing.queryCount;
|
|
6654
|
+
existing.lastQueried = now;
|
|
6655
|
+
existing.hasCompoundIndex = hasCompoundIndex;
|
|
6656
|
+
} else {
|
|
6657
|
+
if (this.compoundStats.size >= this.maxTrackedPatterns) {
|
|
6658
|
+
this.evictOldestCompound();
|
|
6659
|
+
}
|
|
6660
|
+
this.compoundStats.set(compoundKey, {
|
|
6661
|
+
attributes: sortedAttrs,
|
|
6662
|
+
compoundKey,
|
|
6663
|
+
queryCount: this.samplingRate,
|
|
6664
|
+
// Adjust for sampling
|
|
6665
|
+
totalCost: executionTime * this.samplingRate,
|
|
6666
|
+
averageCost: executionTime,
|
|
6667
|
+
lastQueried: now,
|
|
6668
|
+
hasCompoundIndex
|
|
6669
|
+
});
|
|
6670
|
+
}
|
|
6671
|
+
}
|
|
6672
|
+
/**
|
|
6673
|
+
* Get all compound query statistics (Phase 9.03).
|
|
6674
|
+
*
|
|
6675
|
+
* @returns Array of compound query statistics, sorted by query count descending
|
|
6676
|
+
*/
|
|
6677
|
+
getCompoundStatistics() {
|
|
6678
|
+
this.pruneStaleCompound();
|
|
6679
|
+
return Array.from(this.compoundStats.values()).sort((a, b) => b.queryCount - a.queryCount);
|
|
6680
|
+
}
|
|
6681
|
+
/**
|
|
6682
|
+
* Get compound statistics for a specific attribute combination.
|
|
6683
|
+
*
|
|
6684
|
+
* @param attributes - Array of attribute names
|
|
6685
|
+
* @returns Compound query statistics or undefined
|
|
6686
|
+
*/
|
|
6687
|
+
getCompoundStats(attributes) {
|
|
6688
|
+
const sortedAttrs = [...attributes].sort();
|
|
6689
|
+
const compoundKey = sortedAttrs.join("+");
|
|
6690
|
+
return this.compoundStats.get(compoundKey);
|
|
6691
|
+
}
|
|
6692
|
+
/**
|
|
6693
|
+
* Check if attributes appear in any tracked compound queries.
|
|
6694
|
+
*
|
|
6695
|
+
* @param attribute - The attribute name to check
|
|
6696
|
+
* @returns True if attribute is part of any compound query pattern
|
|
6697
|
+
*/
|
|
6698
|
+
isInCompoundPattern(attribute) {
|
|
6699
|
+
for (const stat of this.compoundStats.values()) {
|
|
6700
|
+
if (stat.attributes.includes(attribute)) {
|
|
6701
|
+
return true;
|
|
6702
|
+
}
|
|
6703
|
+
}
|
|
6704
|
+
return false;
|
|
6705
|
+
}
|
|
6706
|
+
/**
|
|
6707
|
+
* Update compound index status.
|
|
6708
|
+
*
|
|
6709
|
+
* @param attributes - Array of attribute names
|
|
6710
|
+
* @param hasCompoundIndex - Whether a compound index exists
|
|
6711
|
+
*/
|
|
6712
|
+
updateCompoundIndexStatus(attributes, hasCompoundIndex) {
|
|
6713
|
+
const sortedAttrs = [...attributes].sort();
|
|
6714
|
+
const compoundKey = sortedAttrs.join("+");
|
|
6715
|
+
const stat = this.compoundStats.get(compoundKey);
|
|
6716
|
+
if (stat) {
|
|
6717
|
+
stat.hasCompoundIndex = hasCompoundIndex;
|
|
6718
|
+
}
|
|
6719
|
+
}
|
|
5879
6720
|
/**
|
|
5880
6721
|
* Get all query statistics.
|
|
5881
6722
|
*
|
|
@@ -5973,6 +6814,7 @@ var QueryPatternTracker = class {
|
|
|
5973
6814
|
*/
|
|
5974
6815
|
clear() {
|
|
5975
6816
|
this.stats.clear();
|
|
6817
|
+
this.compoundStats.clear();
|
|
5976
6818
|
this.queryCounter = 0;
|
|
5977
6819
|
}
|
|
5978
6820
|
/**
|
|
@@ -5981,9 +6823,10 @@ var QueryPatternTracker = class {
|
|
|
5981
6823
|
* @returns Tracking overhead info
|
|
5982
6824
|
*/
|
|
5983
6825
|
getTrackingInfo() {
|
|
5984
|
-
const memoryEstimate = this.stats.size * 200;
|
|
6826
|
+
const memoryEstimate = this.stats.size * 200 + this.compoundStats.size * 300;
|
|
5985
6827
|
return {
|
|
5986
6828
|
patternsTracked: this.stats.size,
|
|
6829
|
+
compoundPatternsTracked: this.compoundStats.size,
|
|
5987
6830
|
totalQueries: this.queryCounter,
|
|
5988
6831
|
samplingRate: this.samplingRate,
|
|
5989
6832
|
memoryEstimate
|
|
@@ -6016,6 +6859,33 @@ var QueryPatternTracker = class {
|
|
|
6016
6859
|
}
|
|
6017
6860
|
}
|
|
6018
6861
|
}
|
|
6862
|
+
/**
|
|
6863
|
+
* Evict the oldest compound query entry (Phase 9.03).
|
|
6864
|
+
*/
|
|
6865
|
+
evictOldestCompound() {
|
|
6866
|
+
let oldestKey = null;
|
|
6867
|
+
let oldestTime = Infinity;
|
|
6868
|
+
for (const [key, stat] of this.compoundStats.entries()) {
|
|
6869
|
+
if (stat.lastQueried < oldestTime) {
|
|
6870
|
+
oldestTime = stat.lastQueried;
|
|
6871
|
+
oldestKey = key;
|
|
6872
|
+
}
|
|
6873
|
+
}
|
|
6874
|
+
if (oldestKey) {
|
|
6875
|
+
this.compoundStats.delete(oldestKey);
|
|
6876
|
+
}
|
|
6877
|
+
}
|
|
6878
|
+
/**
|
|
6879
|
+
* Prune stale compound statistics (Phase 9.03).
|
|
6880
|
+
*/
|
|
6881
|
+
pruneStaleCompound() {
|
|
6882
|
+
const cutoff = Date.now() - this.statsTtl;
|
|
6883
|
+
for (const [key, stat] of this.compoundStats.entries()) {
|
|
6884
|
+
if (stat.lastQueried < cutoff) {
|
|
6885
|
+
this.compoundStats.delete(key);
|
|
6886
|
+
}
|
|
6887
|
+
}
|
|
6888
|
+
}
|
|
6019
6889
|
};
|
|
6020
6890
|
|
|
6021
6891
|
// src/query/adaptive/IndexAdvisor.ts
|
|
@@ -6049,6 +6919,12 @@ var IndexAdvisor = class {
|
|
|
6049
6919
|
suggestions.push(suggestion);
|
|
6050
6920
|
}
|
|
6051
6921
|
}
|
|
6922
|
+
const compoundSuggestions = this.getCompoundSuggestions({
|
|
6923
|
+
minQueryCount,
|
|
6924
|
+
minAverageCost,
|
|
6925
|
+
excludeExistingIndexes
|
|
6926
|
+
});
|
|
6927
|
+
suggestions.push(...compoundSuggestions);
|
|
6052
6928
|
suggestions.sort((a, b) => {
|
|
6053
6929
|
const priorityOrder = { high: 3, medium: 2, low: 1 };
|
|
6054
6930
|
const priorityDiff = priorityOrder[b.priority] - priorityOrder[a.priority];
|
|
@@ -6248,6 +7124,136 @@ var IndexAdvisor = class {
|
|
|
6248
7124
|
}
|
|
6249
7125
|
return reason;
|
|
6250
7126
|
}
|
|
7127
|
+
// ========================================
|
|
7128
|
+
// Phase 9.03: Compound Index Suggestions
|
|
7129
|
+
// ========================================
|
|
7130
|
+
/**
|
|
7131
|
+
* Get compound index suggestions based on AND query patterns.
|
|
7132
|
+
*
|
|
7133
|
+
* @param options - Suggestion options
|
|
7134
|
+
* @returns Array of compound index suggestions
|
|
7135
|
+
*/
|
|
7136
|
+
getCompoundSuggestions(options = {}) {
|
|
7137
|
+
const {
|
|
7138
|
+
minQueryCount = ADAPTIVE_INDEXING_DEFAULTS.advisor.minQueryCount,
|
|
7139
|
+
minAverageCost = ADAPTIVE_INDEXING_DEFAULTS.advisor.minAverageCost,
|
|
7140
|
+
excludeExistingIndexes = true
|
|
7141
|
+
} = options;
|
|
7142
|
+
const compoundStats = this.tracker.getCompoundStatistics();
|
|
7143
|
+
const suggestions = [];
|
|
7144
|
+
for (const stat of compoundStats) {
|
|
7145
|
+
if (excludeExistingIndexes && stat.hasCompoundIndex) continue;
|
|
7146
|
+
if (stat.queryCount < minQueryCount) continue;
|
|
7147
|
+
if (stat.averageCost < minAverageCost) continue;
|
|
7148
|
+
const suggestion = this.generateCompoundSuggestion(stat);
|
|
7149
|
+
if (suggestion) {
|
|
7150
|
+
suggestions.push(suggestion);
|
|
7151
|
+
}
|
|
7152
|
+
}
|
|
7153
|
+
return suggestions;
|
|
7154
|
+
}
|
|
7155
|
+
/**
|
|
7156
|
+
* Get a suggestion for a specific compound attribute combination.
|
|
7157
|
+
*
|
|
7158
|
+
* @param attributes - Array of attribute names
|
|
7159
|
+
* @returns Compound index suggestion or null if not recommended
|
|
7160
|
+
*/
|
|
7161
|
+
getCompoundSuggestionFor(attributes) {
|
|
7162
|
+
const stat = this.tracker.getCompoundStats(attributes);
|
|
7163
|
+
if (!stat) return null;
|
|
7164
|
+
return this.generateCompoundSuggestion(stat);
|
|
7165
|
+
}
|
|
7166
|
+
/**
|
|
7167
|
+
* Check if a compound index should be created for the given attributes.
|
|
7168
|
+
*
|
|
7169
|
+
* @param attributes - Array of attribute names
|
|
7170
|
+
* @param threshold - Minimum query count threshold
|
|
7171
|
+
* @returns True if compound index should be created
|
|
7172
|
+
*/
|
|
7173
|
+
shouldCreateCompoundIndex(attributes, threshold = ADAPTIVE_INDEXING_DEFAULTS.autoIndex.threshold) {
|
|
7174
|
+
const stat = this.tracker.getCompoundStats(attributes);
|
|
7175
|
+
if (!stat) return false;
|
|
7176
|
+
return !stat.hasCompoundIndex && stat.queryCount >= threshold;
|
|
7177
|
+
}
|
|
7178
|
+
/**
|
|
7179
|
+
* Generate a suggestion for a compound query pattern.
|
|
7180
|
+
*/
|
|
7181
|
+
generateCompoundSuggestion(stat) {
|
|
7182
|
+
const estimatedBenefit = this.estimateCompoundBenefit(stat);
|
|
7183
|
+
const estimatedCost = this.estimateCompoundMemoryCost(stat);
|
|
7184
|
+
const priority = this.calculateCompoundPriority(stat, estimatedBenefit);
|
|
7185
|
+
return {
|
|
7186
|
+
attribute: stat.compoundKey,
|
|
7187
|
+
indexType: "compound",
|
|
7188
|
+
reason: this.generateCompoundReason(stat, estimatedBenefit),
|
|
7189
|
+
estimatedBenefit,
|
|
7190
|
+
estimatedCost,
|
|
7191
|
+
priority,
|
|
7192
|
+
queryCount: stat.queryCount,
|
|
7193
|
+
averageCost: stat.averageCost,
|
|
7194
|
+
compoundAttributes: stat.attributes
|
|
7195
|
+
};
|
|
7196
|
+
}
|
|
7197
|
+
/**
|
|
7198
|
+
* Estimate performance benefit of adding a compound index.
|
|
7199
|
+
*
|
|
7200
|
+
* Compound indexes provide significant speedup for AND queries:
|
|
7201
|
+
* - Eliminates intersection operations (100-1000× for each attribute)
|
|
7202
|
+
* - Single O(1) lookup instead of multiple index scans
|
|
7203
|
+
*/
|
|
7204
|
+
estimateCompoundBenefit(stat) {
|
|
7205
|
+
const attributeMultiplier = Math.pow(2, stat.attributes.length - 1);
|
|
7206
|
+
let baseBenefit;
|
|
7207
|
+
if (stat.averageCost > 20) {
|
|
7208
|
+
baseBenefit = 1e3;
|
|
7209
|
+
} else if (stat.averageCost > 5) {
|
|
7210
|
+
baseBenefit = 500;
|
|
7211
|
+
} else if (stat.averageCost > 1) {
|
|
7212
|
+
baseBenefit = 100;
|
|
7213
|
+
} else {
|
|
7214
|
+
baseBenefit = 50;
|
|
7215
|
+
}
|
|
7216
|
+
const frequencyMultiplier = Math.min(stat.queryCount / 10, 100);
|
|
7217
|
+
return Math.floor(baseBenefit * attributeMultiplier * frequencyMultiplier);
|
|
7218
|
+
}
|
|
7219
|
+
/**
|
|
7220
|
+
* Estimate memory cost of adding a compound index.
|
|
7221
|
+
*/
|
|
7222
|
+
estimateCompoundMemoryCost(stat) {
|
|
7223
|
+
const bytesPerRecord = MEMORY_OVERHEAD_ESTIMATES.compound;
|
|
7224
|
+
const attributeOverhead = stat.attributes.length * 8;
|
|
7225
|
+
const estimatedRecords = 1e3;
|
|
7226
|
+
return Math.floor(estimatedRecords * (bytesPerRecord + attributeOverhead) * 1.5);
|
|
7227
|
+
}
|
|
7228
|
+
/**
|
|
7229
|
+
* Calculate priority for compound index suggestion.
|
|
7230
|
+
*/
|
|
7231
|
+
calculateCompoundPriority(stat, estimatedBenefit) {
|
|
7232
|
+
if (stat.queryCount > 100 && stat.averageCost > 10) {
|
|
7233
|
+
return "high";
|
|
7234
|
+
}
|
|
7235
|
+
if (stat.queryCount > 500) {
|
|
7236
|
+
return "high";
|
|
7237
|
+
}
|
|
7238
|
+
if (stat.queryCount > 50 || stat.averageCost > 5) {
|
|
7239
|
+
return "medium";
|
|
7240
|
+
}
|
|
7241
|
+
if (estimatedBenefit > 2e3) {
|
|
7242
|
+
return "medium";
|
|
7243
|
+
}
|
|
7244
|
+
return "low";
|
|
7245
|
+
}
|
|
7246
|
+
/**
|
|
7247
|
+
* Generate human-readable reason for compound index suggestion.
|
|
7248
|
+
*/
|
|
7249
|
+
generateCompoundReason(stat, benefit) {
|
|
7250
|
+
const costStr = stat.averageCost.toFixed(2);
|
|
7251
|
+
const attrList = stat.attributes.join(", ");
|
|
7252
|
+
let reason = `Compound AND query on [${attrList}] executed ${stat.queryCount}\xD7 with average cost ${costStr}ms. `;
|
|
7253
|
+
reason += `Expected ~${benefit}\xD7 cumulative speedup with compound index. `;
|
|
7254
|
+
reason += `Eliminates ${stat.attributes.length - 1} ResultSet intersection(s).`;
|
|
7255
|
+
return reason;
|
|
7256
|
+
}
|
|
6251
7257
|
};
|
|
6252
7258
|
|
|
6253
7259
|
// src/query/adaptive/AutoIndexManager.ts
|
|
@@ -6733,11 +7739,20 @@ var IndexedLWWMap = class extends LWWMap {
|
|
|
6733
7739
|
// ==================== Index Management ====================
|
|
6734
7740
|
/**
|
|
6735
7741
|
* Add a hash index on an attribute.
|
|
7742
|
+
* If lazyIndexBuilding is enabled, creates a LazyHashIndex instead.
|
|
6736
7743
|
*
|
|
6737
7744
|
* @param attribute - Attribute to index
|
|
6738
|
-
* @returns Created HashIndex
|
|
7745
|
+
* @returns Created HashIndex (or LazyHashIndex)
|
|
6739
7746
|
*/
|
|
6740
7747
|
addHashIndex(attribute) {
|
|
7748
|
+
if (this.options.lazyIndexBuilding) {
|
|
7749
|
+
const index2 = new LazyHashIndex(attribute, {
|
|
7750
|
+
onProgress: this.options.onIndexBuilding
|
|
7751
|
+
});
|
|
7752
|
+
this.indexRegistry.addIndex(index2);
|
|
7753
|
+
this.buildIndex(index2);
|
|
7754
|
+
return index2;
|
|
7755
|
+
}
|
|
6741
7756
|
const index = new HashIndex(attribute);
|
|
6742
7757
|
this.indexRegistry.addIndex(index);
|
|
6743
7758
|
this.buildIndex(index);
|
|
@@ -6746,12 +7761,21 @@ var IndexedLWWMap = class extends LWWMap {
|
|
|
6746
7761
|
/**
|
|
6747
7762
|
* Add a navigable index on an attribute.
|
|
6748
7763
|
* Navigable indexes support range queries (gt, gte, lt, lte, between).
|
|
7764
|
+
* If lazyIndexBuilding is enabled, creates a LazyNavigableIndex instead.
|
|
6749
7765
|
*
|
|
6750
7766
|
* @param attribute - Attribute to index
|
|
6751
7767
|
* @param comparator - Optional custom comparator
|
|
6752
|
-
* @returns Created NavigableIndex
|
|
7768
|
+
* @returns Created NavigableIndex (or LazyNavigableIndex)
|
|
6753
7769
|
*/
|
|
6754
7770
|
addNavigableIndex(attribute, comparator) {
|
|
7771
|
+
if (this.options.lazyIndexBuilding) {
|
|
7772
|
+
const index2 = new LazyNavigableIndex(attribute, comparator, {
|
|
7773
|
+
onProgress: this.options.onIndexBuilding
|
|
7774
|
+
});
|
|
7775
|
+
this.indexRegistry.addIndex(index2);
|
|
7776
|
+
this.buildIndex(index2);
|
|
7777
|
+
return index2;
|
|
7778
|
+
}
|
|
6755
7779
|
const index = new NavigableIndex(attribute, comparator);
|
|
6756
7780
|
this.indexRegistry.addIndex(index);
|
|
6757
7781
|
this.buildIndex(index);
|
|
@@ -6760,10 +7784,11 @@ var IndexedLWWMap = class extends LWWMap {
|
|
|
6760
7784
|
/**
|
|
6761
7785
|
* Add an inverted index for full-text search on an attribute.
|
|
6762
7786
|
* Inverted indexes support text search queries (contains, containsAll, containsAny).
|
|
7787
|
+
* If lazyIndexBuilding is enabled, creates a LazyInvertedIndex instead.
|
|
6763
7788
|
*
|
|
6764
7789
|
* @param attribute - Text attribute to index
|
|
6765
7790
|
* @param pipeline - Optional custom tokenization pipeline
|
|
6766
|
-
* @returns Created InvertedIndex
|
|
7791
|
+
* @returns Created InvertedIndex (or LazyInvertedIndex)
|
|
6767
7792
|
*
|
|
6768
7793
|
* @example
|
|
6769
7794
|
* ```typescript
|
|
@@ -6775,6 +7800,14 @@ var IndexedLWWMap = class extends LWWMap {
|
|
|
6775
7800
|
* ```
|
|
6776
7801
|
*/
|
|
6777
7802
|
addInvertedIndex(attribute, pipeline) {
|
|
7803
|
+
if (this.options.lazyIndexBuilding) {
|
|
7804
|
+
const index2 = new LazyInvertedIndex(attribute, pipeline, {
|
|
7805
|
+
onProgress: this.options.onIndexBuilding
|
|
7806
|
+
});
|
|
7807
|
+
this.indexRegistry.addIndex(index2);
|
|
7808
|
+
this.buildIndex(index2);
|
|
7809
|
+
return index2;
|
|
7810
|
+
}
|
|
6778
7811
|
const index = new InvertedIndex(attribute, pipeline);
|
|
6779
7812
|
this.indexRegistry.addIndex(index);
|
|
6780
7813
|
this.buildIndex(index);
|
|
@@ -7279,6 +8312,58 @@ var IndexedLWWMap = class extends LWWMap {
|
|
|
7279
8312
|
isAutoIndexingEnabled() {
|
|
7280
8313
|
return this.autoIndexManager !== null;
|
|
7281
8314
|
}
|
|
8315
|
+
// ==================== Lazy Indexing (Phase 9.01) ====================
|
|
8316
|
+
/**
|
|
8317
|
+
* Check if lazy index building is enabled.
|
|
8318
|
+
*/
|
|
8319
|
+
isLazyIndexingEnabled() {
|
|
8320
|
+
return this.options.lazyIndexBuilding === true;
|
|
8321
|
+
}
|
|
8322
|
+
/**
|
|
8323
|
+
* Force materialization of all lazy indexes.
|
|
8324
|
+
* Useful to pre-warm indexes before critical operations.
|
|
8325
|
+
*
|
|
8326
|
+
* @param progressCallback - Optional progress callback
|
|
8327
|
+
*/
|
|
8328
|
+
materializeAllIndexes(progressCallback) {
|
|
8329
|
+
const callback = progressCallback ?? this.options.onIndexBuilding;
|
|
8330
|
+
for (const index of this.indexRegistry.getAllIndexes()) {
|
|
8331
|
+
if ("isLazy" in index && index.isLazy) {
|
|
8332
|
+
const lazyIndex = index;
|
|
8333
|
+
if (!lazyIndex.isBuilt) {
|
|
8334
|
+
lazyIndex.materialize(callback);
|
|
8335
|
+
}
|
|
8336
|
+
}
|
|
8337
|
+
}
|
|
8338
|
+
}
|
|
8339
|
+
/**
|
|
8340
|
+
* Get count of pending records across all lazy indexes.
|
|
8341
|
+
* Returns 0 if no lazy indexes or all are materialized.
|
|
8342
|
+
*/
|
|
8343
|
+
getPendingIndexCount() {
|
|
8344
|
+
let total = 0;
|
|
8345
|
+
for (const index of this.indexRegistry.getAllIndexes()) {
|
|
8346
|
+
if ("isLazy" in index && index.isLazy) {
|
|
8347
|
+
const lazyIndex = index;
|
|
8348
|
+
total += lazyIndex.pendingCount;
|
|
8349
|
+
}
|
|
8350
|
+
}
|
|
8351
|
+
return total;
|
|
8352
|
+
}
|
|
8353
|
+
/**
|
|
8354
|
+
* Check if any lazy indexes are still pending (not built).
|
|
8355
|
+
*/
|
|
8356
|
+
hasUnbuiltIndexes() {
|
|
8357
|
+
for (const index of this.indexRegistry.getAllIndexes()) {
|
|
8358
|
+
if ("isLazy" in index && index.isLazy) {
|
|
8359
|
+
const lazyIndex = index;
|
|
8360
|
+
if (!lazyIndex.isBuilt) {
|
|
8361
|
+
return true;
|
|
8362
|
+
}
|
|
8363
|
+
}
|
|
8364
|
+
}
|
|
8365
|
+
return false;
|
|
8366
|
+
}
|
|
7282
8367
|
/**
|
|
7283
8368
|
* Track query pattern for adaptive indexing.
|
|
7284
8369
|
*/
|