document-dataply 0.0.10-alpha.3 → 0.0.10-alpha.5

package/dist/types/core/AnalysisManager.d.ts

@@ -0,0 +1,105 @@
+import type { AnalysisHeader, DocumentJSON, FlattenedDocumentJSON } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import type { AnalysisProvider } from './AnalysisProvider';
+import { Transaction } from 'dataply';
+export declare class AnalysisManager<T extends DocumentJSON> {
+    private api;
+    private providers;
+    constructor(api: DocumentDataplyAPI<T>);
+    /**
+     * Register all built-in analysis providers.
+     * Each provider class is instantiated with the API reference and registered.
+     */
+    registerBuiltinProviders(): void;
+    /**
+     * Register an analysis provider.
+     * @param provider The provider instance to register
+     */
+    registerProvider(provider: AnalysisProvider<T>): void;
+    /**
+     * Get a registered analysis provider by name.
+     * @param name The provider name
+     * @returns The provider instance, or null if not found
+     */
+    getProvider<P extends AnalysisProvider<T> = AnalysisProvider<T>>(name: string): P | null;
+    /**
+     * Initialize all registered providers by loading existing data from disk.
+     * Should be called after database initialization.
+     * @param tx The transaction to use
+     */
+    initializeProviders(tx: Transaction): Promise<void>;
+    /**
+     * Notify all realtime providers that documents were inserted.
+     * Data is persisted immediately after each provider processes the mutation.
+     * @param documents The flattened documents that were inserted
+     * @param tx The transaction to use
+     */
+    notifyInsert(documents: FlattenedDocumentJSON[], tx: Transaction): Promise<void>;
+    /**
+     * Notify all realtime providers that documents were deleted.
+     * Data is persisted immediately after each provider processes the mutation.
+     * @param documents The flattened documents that were deleted
+     * @param tx The transaction to use
+     */
+    notifyDelete(documents: FlattenedDocumentJSON[], tx: Transaction): Promise<void>;
+    /**
+     * Notify all realtime providers that documents were updated.
+     * Data is persisted immediately after each provider processes the mutation.
+     * @param pairs Array of { oldDocument, newDocument } pairs
+     * @param tx The transaction to use
+     */
+    notifyUpdate(pairs: {
+        oldDocument: FlattenedDocumentJSON;
+        newDocument: FlattenedDocumentJSON;
+    }[], tx: Transaction): Promise<void>;
+    /**
+     * Flush all interval providers' data to disk.
+     * @param tx The transaction to use (must be a write transaction)
+     */
+    flush(tx: Transaction): Promise<void>;
+    /**
+     * Get the analysis header row.
+     * Returns null if no analysis header exists yet.
+     * @param tx The transaction to use
+     */
+    getAnalysisHeader(tx: Transaction): Promise<AnalysisHeader | null>;
+    /**
+     * Get the analysis header row, creating it if it doesn't exist.
+     * @param tx The transaction to use (must be a write transaction)
+     */
+    getOrCreateAnalysisHeader(tx: Transaction): Promise<AnalysisHeader>;
+    /**
+     * Get analysis data for a specific type as a raw string.
+     * Returns null if the type doesn't exist in the analysis header.
+     * @param type The analysis type name
+     * @param tx The transaction to use
+     */
+    getAnalysisData(type: string, tx: Transaction): Promise<string | null>;
+    /**
+     * Set analysis data for a specific type.
+     * Creates a new overflow row if the type doesn't exist yet,
+     * or updates the existing row if it does.
+     * @param type The analysis type name
+     * @param data The raw string data to store
+     * @param tx The transaction to use (must be a write transaction)
+     */
+    setAnalysisData(type: string, data: string, tx: Transaction): Promise<void>;
+    /**
+     * Delete analysis data for a specific type.
+     * Removes the type entry from the analysis header.
+     * @param type The analysis type name
+     * @param tx The transaction to use (must be a write transaction)
+     */
+    deleteAnalysisData(type: string, tx: Transaction): Promise<boolean>;
+    /**
+     * Check if analysis data exists for a specific type.
+     * @param type The analysis type name
+     * @param tx The transaction to use
+     */
+    hasAnalysisData(type: string, tx: Transaction): Promise<boolean>;
+    /**
+     * Get all registered analysis type names.
+     * @param tx The transaction to use
+     */
+    getAnalysisTypes(tx: Transaction): Promise<string[]>;
+}

package/dist/types/core/AnalysisProvider.d.ts

@@ -0,0 +1,30 @@
+import type { DocumentJSON } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import type { Transaction } from 'dataply';
+/**
+ * Abstract base class for analysis providers.
+ * Subclasses should extend either RealtimeAnalysisProvider or IntervalAnalysisProvider.
+ */
+export declare abstract class AnalysisProvider<T extends DocumentJSON = DocumentJSON> {
+    protected api: DocumentDataplyAPI<T>;
+    /** Overflow row PK assigned by AnalysisManager during initialization. */
+    storageKey: number;
+    constructor(api: DocumentDataplyAPI<T>);
+    /**
+     * Unique name of this analysis type (e.g. 'ftsTermCount').
+     * Used as the key in the AnalysisHeader.
+     */
+    abstract readonly name: string;
+    /**
+     * Load existing statistics data from a raw string.
+     * Called during initialization to restore state from disk.
+     * @param raw The raw string from the overflow row, or null if no data exists yet
+     * @param tx Optional transaction
+     */
+    abstract load(raw: string | null, tx: Transaction): Promise<void>;
+    /**
+     * Serialize the current statistics data to a raw string for storage.
+     * @param tx Optional transaction
+     */
+    abstract serialize(tx: Transaction): Promise<string>;
+}

package/dist/types/core/DocumentFormatter.d.ts

@@ -0,0 +1,5 @@
+import type { DocumentJSON, FlattenedDocumentJSON } from '../types';
+export declare class DocumentFormatter<T extends DocumentJSON> {
+    private flattenInternal;
+    flattenDocument(document: T): FlattenedDocumentJSON;
+}

package/dist/types/core/IndexManager.d.ts

@@ -0,0 +1,68 @@
+import type { DataplyTreeValue, DocumentDataplyInnerMetadata, Primitive, CreateIndexOption, IndexMetaConfig, FTSConfig, DocumentJSON, FlattenedDocumentJSON } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import { BPTreeAsync, Transaction } from 'dataply';
+export declare class IndexManager<T extends DocumentJSON> {
+    private api;
+    indices: DocumentDataplyInnerMetadata['indices'];
+    readonly trees: Map<string, BPTreeAsync<string | number, DataplyTreeValue<Primitive>>>;
+    readonly indexedFields: Set<string>;
+    /**
+     * Registered indices via createIndex() (before init)
+     * Key: index name, Value: index configuration
+     */
+    readonly pendingCreateIndices: Map<string, CreateIndexOption<T>>;
+    /**
+     * Resolved index configurations after init.
+     * Key: index name, Value: index config (from metadata)
+     */
+    registeredIndices: Map<string, IndexMetaConfig>;
+    /**
+     * Maps field name → index names that cover this field.
+     * Used for query resolution.
+     */
+    fieldToIndices: Map<string, string[]>;
+    pendingBackfillFields: string[];
+    constructor(api: DocumentDataplyAPI<T>);
+    /**
+     * Validate and apply indices from DB metadata and pending indices.
+     * Called during database initialization.
+     */
+    initializeIndices(metadata: DocumentDataplyInnerMetadata, isNewlyCreated: boolean, tx: Transaction): Promise<boolean>;
+    /**
+     * Register an index. If called before init(), queues it.
+     */
+    registerIndex(name: string, option: CreateIndexOption<T>, tx?: Transaction): Promise<void>;
+    /**
+     * Register an index at runtime (after init).
+     */
+    private registerIndexRuntime;
+    /**
+     * Drop (remove) a named index.
+     */
+    dropIndex(name: string, tx?: Transaction): Promise<void>;
+    /**
+     * Backfill indices for newly created indices after data was inserted.
+     */
+    backfillIndices(tx?: Transaction): Promise<number>;
+    /**
+     * Convert CreateIndexOption to IndexMetaConfig for metadata storage.
+     */
+    toIndexMetaConfig(option: CreateIndexOption<T>): IndexMetaConfig;
+    /**
+     * Get all field names from an IndexMetaConfig.
+     */
+    getFieldsFromConfig(config: IndexMetaConfig): string[];
+    /**
+     * Get the primary field of an index.
+     */
+    getPrimaryField(config: IndexMetaConfig): string;
+    /**
+     * Create B+Tree value string for indexing a document
+     */
+    getIndexValue(config: IndexMetaConfig, flatDoc: FlattenedDocumentJSON): Primitive | Primitive[] | undefined;
+    /**
+     * Get FTSConfig from IndexMetaConfig
+     */
+    getFtsConfig(config: IndexMetaConfig): FTSConfig | null;
+    getTokenKey(pk: number, token: string): string;
+}

package/dist/types/core/IntervalAnalysisProvider.d.ts

@@ -0,0 +1,31 @@
+import type { DocumentJSON, DataplyDocument } from '../types';
+import { AnalysisProvider } from './AnalysisProvider';
+import type { Transaction } from 'dataply';
+/**
+ * Sampling options for interval analysis providers.
+ * Specify either a ratio (0~1) or an exact count.
+ */
+export type SampleOptions = {
+    /** Ratio of documents to sample (0 exclusive ~ 1 inclusive) */
+    rate: number;
+    count?: never;
+} | {
+    rate?: never;
+    /** Exact number of documents to sample */
+    count: number;
+};
+/**
+ * Abstract base class for interval analysis providers.
+ * Data is accumulated in memory and persisted only when flush() is called.
+ * No mutation hooks — state is computed independently (e.g. on a schedule or at init).
+ */
+export declare abstract class IntervalAnalysisProvider<T extends DocumentJSON = DocumentJSON> extends AnalysisProvider<T> {
+    /**
+     * Sample random documents from the entire dataset.
+     * Fetches only PK index, then reads only the selected documents from disk.
+     * @param sampleOptions Sampling strategy — either `{ rate }` or `{ count }`
+     * @param tx Optional transaction
+     * @returns Randomly selected documents
+     */
+    sample(sampleOptions: SampleOptions, tx?: Transaction): Promise<DataplyDocument<T>[]>;
+}

package/dist/types/core/MetadataManager.d.ts

@@ -0,0 +1,11 @@
+import type { DocumentDataplyMetadata, DocumentDataplyInnerMetadata, DocumentJSON } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import { Transaction } from 'dataply';
+export declare class MetadataManager<T extends DocumentJSON> {
+    private api;
+    constructor(api: DocumentDataplyAPI<T>);
+    getDocumentMetadata(tx: Transaction): Promise<DocumentDataplyMetadata>;
+    getDocumentInnerMetadata(tx: Transaction): Promise<DocumentDataplyInnerMetadata>;
+    updateDocumentInnerMetadata(metadata: DocumentDataplyInnerMetadata, tx: Transaction): Promise<void>;
+    migration(version: number, callback: (tx: Transaction) => Promise<void>, tx?: Transaction): Promise<void>;
+}

package/dist/types/core/MutationManager.d.ts

@@ -0,0 +1,14 @@
+import type { DocumentJSON, DataplyDocument, DocumentDataplyQuery } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import { Transaction } from 'dataply';
+export declare class MutationManager<T extends DocumentJSON> {
+    private api;
+    constructor(api: DocumentDataplyAPI<T>);
+    private insertDocumentInternal;
+    insertSingleDocument(document: T, tx?: Transaction): Promise<number>;
+    insertBatchDocuments(documents: T[], tx?: Transaction): Promise<number[]>;
+    private updateInternal;
+    fullUpdate(query: Partial<DocumentDataplyQuery<T>>, newRecord: T | ((document: DataplyDocument<T>) => T), tx?: Transaction): Promise<number>;
+    partialUpdate(query: Partial<DocumentDataplyQuery<T>>, newRecord: Partial<DataplyDocument<T>> | ((document: DataplyDocument<T>) => Partial<DataplyDocument<T>>), tx?: Transaction): Promise<number>;
+    deleteDocuments(query: Partial<DocumentDataplyQuery<T>>, tx?: Transaction): Promise<number>;
+}

package/dist/types/core/Optimizer.d.ts

@@ -0,0 +1,79 @@
+import type { DataplyTreeValue, DocumentDataplyQuery, DocumentDataplyCondition } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import { BPTreeAsync } from 'dataply';
+export declare class Optimizer<T extends Record<string, any>> {
+    private api;
+    constructor(api: DocumentDataplyAPI<T>);
+    /**
+     * B-Tree 타입 인덱스의 선택도를 평가하고 트리에 부여할 조건을 산출합니다.
+     */
+    evaluateBTreeCandidate<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(indexName: string, config: any, query: Partial<DocumentDataplyQuery<V>>, queryFields: Set<string>, treeTx: BPTreeAsync<string | number, V>, orderByField?: string): {
+        readonly tree: BPTreeAsync<string | number, V>;
+        readonly condition: any;
+        readonly field: any;
+        readonly indexName: string;
+        readonly isFtsMatch: false;
+        readonly score: number;
+        readonly compositeVerifyFields: string[];
+        readonly coveredFields: string[];
+        readonly isIndexOrderSupported: boolean;
+    } | null;
+    /**
+     * FTS 타입 인덱스의 선택도를 평가합니다.
+     * FTSTermCount 통계가 있으면 토큰 빈도 기반 동적 score를 산출합니다.
+     */
+    evaluateFTSCandidate<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(indexName: string, config: any, query: Partial<DocumentDataplyQuery<V>>, queryFields: Set<string>, treeTx: BPTreeAsync<string | number, V>): {
+        readonly tree: BPTreeAsync<string | number, V>;
+        readonly condition: any;
+        readonly field: any;
+        readonly indexName: string;
+        readonly isFtsMatch: true;
+        readonly matchTokens: string[];
+        readonly score: number;
+        readonly compositeVerifyFields: readonly [];
+        readonly coveredFields: readonly [any];
+        readonly isIndexOrderSupported: false;
+    } | null;
+    /**
+     * 실행할 최적의 인덱스를 선택합니다. (최적 드라이버 선택)
+     */
+    getSelectivityCandidate<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(query: Partial<DocumentDataplyQuery<V>>, orderByField?: string): Promise<{
+        driver: ({
+            tree: BPTreeAsync<number, V>;
+            condition: Partial<DocumentDataplyCondition<U>>;
+            field: string;
+            indexName: string;
+            isFtsMatch: false;
+            isIndexOrderSupported: boolean;
+        } | {
+            tree: BPTreeAsync<string, V>;
+            condition: Partial<DocumentDataplyCondition<U>>;
+            field: string;
+            indexName: string;
+            isFtsMatch: true;
+            matchTokens: string[];
+            isIndexOrderSupported: boolean;
+        });
+        others: ({
+            tree: BPTreeAsync<number, V>;
+            condition: Partial<DocumentDataplyCondition<U>>;
+            field: string;
+            indexName: string;
+            isFtsMatch: false;
+            isIndexOrderSupported: boolean;
+        } | {
+            tree: BPTreeAsync<string, V>;
+            condition: Partial<DocumentDataplyCondition<U>>;
+            field: string;
+            indexName: string;
+            isFtsMatch: true;
+            matchTokens: string[];
+            isIndexOrderSupported: boolean;
+        })[];
+        compositeVerifyConditions: {
+            field: string;
+            condition: any;
+        }[];
+        rollback: () => void;
+    } | null>;
+}

package/dist/types/core/QueryManager.d.ts

@@ -0,0 +1,75 @@
+import type { DataplyTreeValue, DocumentDataplyQuery, DocumentDataplyQueryOptions, DataplyDocument, Primitive, DocumentJSON } from '../types';
+import type { DocumentDataplyAPI } from './documentAPI';
+import type { Optimizer } from './Optimizer';
+import { BPTreeAsync } from 'dataply';
+export declare class QueryManager<T extends DocumentJSON> {
+    private api;
+    private optimizer;
+    private readonly operatorConverters;
+    constructor(api: DocumentDataplyAPI<T>, optimizer: Optimizer<T>);
+    /**
+     * Transforms a query object into a verbose query object
+     */
+    verboseQuery<U extends Partial<DocumentDataplyQuery<T>>, V extends DataplyTreeValue<U>>(query: Partial<DocumentDataplyQuery<U>>): Partial<DocumentDataplyQuery<V>>;
+    getFreeMemoryChunkSize(): {
+        verySmallChunkSize: number;
+        smallChunkSize: number;
+    };
+    private applyCandidateByFTSStream;
+    private applyCandidateStream;
+    getKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc'): Promise<Float64Array>;
+    getDriverKeys(query: Partial<DocumentDataplyQuery<T>>, orderBy?: string, sortOrder?: 'asc' | 'desc'): Promise<{
+        keysStream: AsyncIterableIterator<number>;
+        others: {
+            tree: BPTreeAsync<string | number, DataplyTreeValue<Primitive>>;
+            condition: any;
+            field: string;
+            indexName: string;
+            isFtsMatch: boolean;
+            matchTokens?: string[];
+            coveredFields?: string[];
+        }[];
+        compositeVerifyConditions: {
+            field: string;
+            condition: any;
+        }[];
+        isDriverOrderByField: boolean;
+        rollback: () => void;
+    } | null>;
+    verifyFts(doc: DataplyDocument<T>, ftsConditions: {
+        field: string;
+        matchTokens: string[];
+    }[]): boolean;
+    verifyCompositeConditions(doc: DataplyDocument<T>, conditions: {
+        field: string;
+        condition: any;
+    }[]): boolean;
+    verifyValue(value: Primitive, condition: any): boolean;
+    adjustChunkSize(currentChunkSize: number, chunkTotalSize: number): number;
+    processChunkedKeysWithVerify(keysStream: AsyncIterableIterator<number>, startIdx: number, initialChunkSize: number, limit: number, ftsConditions: {
+        field: string;
+        matchTokens: string[];
+    }[], compositeVerifyConditions: {
+        field: string;
+        condition: any;
+    }[], others: {
+        tree: BPTreeAsync<string | number, DataplyTreeValue<Primitive>>;
+        condition: any;
+        field: string;
+        indexName: string;
+        isFtsMatch: boolean;
+        matchTokens?: string[];
+        coveredFields?: string[];
+    }[], tx: any): AsyncGenerator<DataplyDocument<T>>;
+    /**
+     * Count documents from the database that match the query
+     * @param query The query to use
+     * @param tx The transaction to use
+     * @returns The number of documents that match the query
+     */
+    countDocuments(query: Partial<DocumentDataplyQuery<T>>, tx?: any): Promise<number>;
+    selectDocuments(query: Partial<DocumentDataplyQuery<T>>, options?: DocumentDataplyQueryOptions, tx?: any): {
+        stream: () => AsyncIterableIterator<DataplyDocument<T>>;
+        drain: () => Promise<DataplyDocument<T>[]>;
+    };
+}

package/dist/types/core/RealtimeAnalysisProvider.d.ts

@@ -0,0 +1,27 @@
+import type { FlattenedDocumentJSON, DocumentJSON } from '../types';
+import { AnalysisProvider } from './AnalysisProvider';
+/**
+ * Abstract base class for realtime analysis providers.
+ * Mutation hooks (onInsert, onDelete, onUpdate) are called on every mutation
+ * and the result is persisted immediately.
+ */
+export declare abstract class RealtimeAnalysisProvider<T extends DocumentJSON = DocumentJSON> extends AnalysisProvider<T> {
+    /**
+     * Called when documents are inserted.
+     * @param documents The flattened documents that were inserted
+     */
+    abstract onInsert(documents: FlattenedDocumentJSON[]): Promise<void>;
+    /**
+     * Called when documents are deleted.
+     * @param documents The flattened documents that were deleted
+     */
+    abstract onDelete(documents: FlattenedDocumentJSON[]): Promise<void>;
+    /**
+     * Called when documents are updated.
+     * @param pairs Array of { oldDocument, newDocument } pairs
+     */
+    abstract onUpdate(pairs: {
+        oldDocument: FlattenedDocumentJSON;
+        newDocument: FlattenedDocumentJSON;
+    }[]): Promise<void>;
+}

package/dist/types/core/analysis/FTSTermCount.d.ts

@@ -0,0 +1,28 @@
+import type { DocumentJSON } from '../../types';
+import { Transaction } from 'dataply';
+import { IntervalAnalysisProvider } from '../IntervalAnalysisProvider';
+export declare class FTSTermCount<T extends DocumentJSON = DocumentJSON> extends IntervalAnalysisProvider<T> {
+    readonly name = "fts_term_count";
+    private termCount;
+    private sampleSize;
+    serialize(tx: Transaction): Promise<string>;
+    load(data: string | null, tx: Transaction): Promise<void>;
+    /**
+     * 특정 field/strategy/token의 문서 빈도를 반환합니다.
+     * 통계에 없으면 0을 반환합니다.
+     */
+    getTermCount(field: string, strategy: string, token: string): number;
+    /**
+     * 쿼리 토큰 배열에서 최소 빈도(AND 시맨틱스 상한선)를 반환합니다.
+     * 통계가 없거나 sampleSize가 0이면 -1을 반환합니다.
+     */
+    getMinTokenCount(field: string, strategy: string, tokens: string[]): number;
+    /**
+     * 통계가 유효한지 여부를 반환합니다.
+     */
+    get hasSampleData(): boolean;
+    /**
+     * 통계 수집 시 사용된 샘플 크기를 반환합니다.
+     */
+    getSampleSize(): number;
+}

package/dist/types/core/analysis/index.d.ts

@@ -0,0 +1,2 @@
+import { FTSTermCount } from './FTSTermCount';
+export declare const BuiltinAnalysisProviders: (typeof FTSTermCount)[];

package/dist/types/core/document.d.ts

@@ -52,6 +52,12 @@ export declare class DocumentDataply<T extends DocumentJSON> {
      * Initialize the document database
      */
     init(): Promise<void>;
+    /**
+     * Flush all interval analysis providers, forcing statistics to be recalculated.
+     * Call this after bulk inserts or periodically to keep FTS statistics fresh.
+     * @param tx Optional transaction
+     */
+    flushAnalysis(tx?: Transaction): Promise<void>;
     /**
      * Run a migration if the current schemeVersion is lower than the target version.
      * The callback is only executed when the database's schemeVersion is below the given version.