document-dataply 0.0.14-alpha.3 → 0.0.14-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cjs/index.js
CHANGED
|
@@ -14600,11 +14600,19 @@ var E = class {
|
|
|
14600
14600
|
|
|
14601
14601
|
// src/core/AnalysisProvider.ts
|
|
14602
14602
|
var AnalysisProvider = class {
|
|
14603
|
-
constructor(api) {
|
|
14603
|
+
constructor(api, loggerManager) {
|
|
14604
14604
|
this.api = api;
|
|
14605
|
+
this.loggerManager = loggerManager;
|
|
14605
14606
|
}
|
|
14606
14607
|
/** Overflow row PK assigned by AnalysisManager during initialization. */
|
|
14607
14608
|
storageKey = -1;
|
|
14609
|
+
_logger;
|
|
14610
|
+
get logger() {
|
|
14611
|
+
if (!this._logger) {
|
|
14612
|
+
this._logger = this.loggerManager.create(`document-dataply:analysis:${this.name}`);
|
|
14613
|
+
}
|
|
14614
|
+
return this._logger;
|
|
14615
|
+
}
|
|
14608
14616
|
};
|
|
14609
14617
|
|
|
14610
14618
|
// src/core/RealtimeAnalysisProvider.ts
|
|
@@ -14650,6 +14658,7 @@ var FTSTermCount = class extends IntervalAnalysisProvider {
|
|
|
14650
14658
|
termCount = {};
|
|
14651
14659
|
sampleSize = 0;
|
|
14652
14660
|
async serialize(tx) {
|
|
14661
|
+
this.logger.debug(`Starting serialize`);
|
|
14653
14662
|
const docs = await this.sample({ count: this.api.analysisManager.sampleSize }, tx);
|
|
14654
14663
|
this.termCount = {};
|
|
14655
14664
|
this.sampleSize = docs.length;
|
|
@@ -14698,12 +14707,15 @@ var FTSTermCount = class extends IntervalAnalysisProvider {
|
|
|
14698
14707
|
}
|
|
14699
14708
|
}
|
|
14700
14709
|
this.termCount = optimizedTermCount;
|
|
14710
|
+
this.logger.debug(`Serialize complete, sampleSize: ${this.sampleSize}`);
|
|
14701
14711
|
return JSON.stringify({ _sampleSize: this.sampleSize, ...this.termCount });
|
|
14702
14712
|
}
|
|
14703
14713
|
async load(data, tx) {
|
|
14714
|
+
this.logger.debug(`Loading data`);
|
|
14704
14715
|
this.termCount = {};
|
|
14705
14716
|
this.sampleSize = 0;
|
|
14706
14717
|
if (!data) {
|
|
14718
|
+
this.logger.debug(`No existing data found, initialized as empty`);
|
|
14707
14719
|
return;
|
|
14708
14720
|
}
|
|
14709
14721
|
try {
|
|
@@ -14713,7 +14725,9 @@ var FTSTermCount = class extends IntervalAnalysisProvider {
|
|
|
14713
14725
|
this.sampleSize = typeof _sampleSize === "number" ? _sampleSize : 0;
|
|
14714
14726
|
this.termCount = rest;
|
|
14715
14727
|
}
|
|
14728
|
+
this.logger.debug(`Successfully parsed existing data (sampleSize: ${this.sampleSize})`);
|
|
14716
14729
|
} catch (e) {
|
|
14730
|
+
this.logger.warn(`Failed to parse existing data`, e);
|
|
14717
14731
|
}
|
|
14718
14732
|
}
|
|
14719
14733
|
/**
|
|
@@ -14757,18 +14771,26 @@ var BuiltinAnalysisProviders = [
|
|
|
14757
14771
|
|
|
14758
14772
|
// src/core/AnalysisManager.ts
|
|
14759
14773
|
var AnalysisManager = class {
|
|
14760
|
-
constructor(api, schedule, sampleSize,
|
|
14774
|
+
constructor(api, schedule, sampleSize, loggerManager) {
|
|
14761
14775
|
this.api = api;
|
|
14762
14776
|
this.sampleSize = sampleSize;
|
|
14763
|
-
this.
|
|
14777
|
+
this.loggerManager = loggerManager;
|
|
14778
|
+
this.logger = loggerManager.create("document-dataply:analysis");
|
|
14764
14779
|
this.cron = new E(schedule, async () => {
|
|
14765
14780
|
if (this.flushing) return;
|
|
14766
14781
|
await this.api.flushAnalysis();
|
|
14782
|
+
}, {
|
|
14783
|
+
paused: true,
|
|
14784
|
+
unref: true,
|
|
14785
|
+
catch: (err) => {
|
|
14786
|
+
this.logger.error("Error in analysis cron job", err);
|
|
14787
|
+
}
|
|
14767
14788
|
});
|
|
14768
14789
|
}
|
|
14769
14790
|
providers = /* @__PURE__ */ new Map();
|
|
14770
14791
|
cron = null;
|
|
14771
14792
|
flushing = false;
|
|
14793
|
+
logger;
|
|
14772
14794
|
/**
|
|
14773
14795
|
* Stop the background analysis cron job.
|
|
14774
14796
|
*/
|
|
@@ -14784,7 +14806,7 @@ var AnalysisManager = class {
|
|
|
14784
14806
|
*/
|
|
14785
14807
|
registerBuiltinProviders() {
|
|
14786
14808
|
for (const Provider of BuiltinAnalysisProviders) {
|
|
14787
|
-
const instance = new Provider(this.api);
|
|
14809
|
+
const instance = new Provider(this.api, this.loggerManager);
|
|
14788
14810
|
this.registerProvider(instance);
|
|
14789
14811
|
}
|
|
14790
14812
|
}
|
|
@@ -14829,11 +14851,11 @@ var AnalysisManager = class {
|
|
|
14829
14851
|
}
|
|
14830
14852
|
}
|
|
14831
14853
|
/**
|
|
14832
|
-
*
|
|
14854
|
+
* Resume the background analysis cron job.
|
|
14833
14855
|
*/
|
|
14834
|
-
|
|
14856
|
+
resumeCron() {
|
|
14835
14857
|
if (this.cron && !this.cron.isRunning()) {
|
|
14836
|
-
this.cron.
|
|
14858
|
+
this.cron.resume();
|
|
14837
14859
|
}
|
|
14838
14860
|
}
|
|
14839
14861
|
/**
|
|
@@ -15032,9 +15054,9 @@ var DocumentDataplyAPI = class extends import_dataply4.DataplyAPI {
|
|
|
15032
15054
|
this.documentFormatter = new DocumentFormatter();
|
|
15033
15055
|
this.analysisManager = new AnalysisManager(
|
|
15034
15056
|
this,
|
|
15035
|
-
options.analysisSchedule ?? "
|
|
15057
|
+
options.analysisSchedule ?? "0 */1 * * *",
|
|
15036
15058
|
options.analysisSampleSize ?? 1e3,
|
|
15037
|
-
this.loggerManager
|
|
15059
|
+
this.loggerManager
|
|
15038
15060
|
);
|
|
15039
15061
|
this.hook.onceAfter("close", async () => {
|
|
15040
15062
|
this.logger.info("DocumentDataplyAPI closing");
|
|
@@ -15055,7 +15077,7 @@ var DocumentDataplyAPI = class extends import_dataply4.DataplyAPI {
|
|
|
15055
15077
|
this.logger.debug(`Indices initialized. Total indices: ${Object.keys(metadata.indices).length}`);
|
|
15056
15078
|
this.analysisManager.registerBuiltinProviders();
|
|
15057
15079
|
await this.analysisManager.initializeProviders(tx);
|
|
15058
|
-
this.analysisManager.
|
|
15080
|
+
this.analysisManager.resumeCron();
|
|
15059
15081
|
this._initialized = true;
|
|
15060
15082
|
this.logger.info("Document database fully initialized");
|
|
15061
15083
|
return tx;
|
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import type { AnalysisHeader, DocumentJSON, FlattenedDocumentJSON } from '../types';
|
|
2
2
|
import type { DocumentDataplyAPI } from './documentAPI';
|
|
3
3
|
import type { AnalysisProvider } from './AnalysisProvider';
|
|
4
|
-
import { Transaction,
|
|
4
|
+
import { Transaction, LoggerManager } from 'dataply';
|
|
5
5
|
export declare class AnalysisManager<T extends DocumentJSON> {
|
|
6
6
|
private api;
|
|
7
7
|
readonly sampleSize: number;
|
|
8
|
-
private
|
|
8
|
+
private loggerManager;
|
|
9
9
|
private providers;
|
|
10
10
|
private cron;
|
|
11
11
|
private flushing;
|
|
12
|
-
|
|
12
|
+
private logger;
|
|
13
|
+
constructor(api: DocumentDataplyAPI<T>, schedule: string, sampleSize: number, loggerManager: LoggerManager);
|
|
13
14
|
/**
|
|
14
15
|
* Stop the background analysis cron job.
|
|
15
16
|
*/
|
|
@@ -37,9 +38,9 @@ export declare class AnalysisManager<T extends DocumentJSON> {
|
|
|
37
38
|
*/
|
|
38
39
|
initializeProviders(tx: Transaction): Promise<void>;
|
|
39
40
|
/**
|
|
40
|
-
*
|
|
41
|
+
* Resume the background analysis cron job.
|
|
41
42
|
*/
|
|
42
|
-
|
|
43
|
+
resumeCron(): void;
|
|
43
44
|
/**
|
|
44
45
|
* Notify all realtime providers that documents were inserted.
|
|
45
46
|
* Data is persisted immediately after each provider processes the mutation.
|
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import type { DocumentJSON } from '../types';
|
|
2
2
|
import type { DocumentDataplyAPI } from './documentAPI';
|
|
3
|
-
import type { Transaction } from 'dataply';
|
|
3
|
+
import type { Transaction, Logger, LoggerManager } from 'dataply';
|
|
4
4
|
/**
|
|
5
5
|
* Abstract base class for analysis providers.
|
|
6
6
|
* Subclasses should extend either RealtimeAnalysisProvider or IntervalAnalysisProvider.
|
|
7
7
|
*/
|
|
8
8
|
export declare abstract class AnalysisProvider<T extends DocumentJSON = DocumentJSON> {
|
|
9
9
|
protected api: DocumentDataplyAPI<T>;
|
|
10
|
+
protected loggerManager: LoggerManager;
|
|
10
11
|
/** Overflow row PK assigned by AnalysisManager during initialization. */
|
|
11
12
|
storageKey: number;
|
|
12
|
-
|
|
13
|
+
private _logger?;
|
|
14
|
+
protected get logger(): Logger;
|
|
15
|
+
constructor(api: DocumentDataplyAPI<T>, loggerManager: LoggerManager);
|
|
13
16
|
/**
|
|
14
17
|
* Unique name of this analysis type (e.g. 'ftsTermCount').
|
|
15
18
|
* Used as the key in the AnalysisHeader.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "document-dataply",
|
|
3
|
-
"version": "0.0.14-alpha.
|
|
3
|
+
"version": "0.0.14-alpha.5",
|
|
4
4
|
"description": "Simple and powerful JSON document database supporting complex queries and flexible indexing policies.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "izure <admin@izure.org>",
|