@ceramicnetwork/indexing 1.1.0-rc.0

package/README.md

@@ -0,0 +1,31 @@
+# Ceramic indexing
+
+## Installation
+
+```
+$ npm install @ceramicnetwork/indexing
+```
+
+## Development
+
+Run tests:
+
+```shell
+npm test
+```
+
+Run linter:
+
+```shell
+npm run lint
+```
+
+## Contributing
+
+We are happy to accept small and large contributions. Make sure to check out the
+[Ceramic specifications](https://github.com/ceramicnetwork/ceramic/blob/main/SPECIFICATION.md)
+for details of how the protocol works.
+
+## License
+
+MIT or Apache-2.0

package/lib/__tests__/chunks.util.d.ts

@@ -0,0 +1,2 @@
+export declare function chunks<A>(array: Array<A>, size: number): Array<Array<A>>;
+//# sourceMappingURL=chunks.util.d.ts.map

package/lib/__tests__/chunks.util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunks.util.d.ts","sourceRoot":"","sources":["../../src/__tests__/chunks.util.ts"],"names":[],"mappings":"AAGA,wBAAgB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAMxE"}

package/lib/__tests__/chunks.util.js

@@ -0,0 +1,8 @@
+export function chunks(array, size) {
+    const results = [];
+    for (let i = 0; i < array.length; i += size) {
+        results.push(array.slice(i, i + size));
+    }
+    return results;
+}
+//# sourceMappingURL=chunks.util.js.map

package/lib/__tests__/chunks.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunks.util.js","sourceRoot":"","sources":["../../src/__tests__/chunks.util.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,MAAM,CAAI,KAAe,EAAE,IAAY;IACrD,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE;QAC3C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;KACvC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC"}

package/lib/__tests__/read-csv-fixture.util.d.ts

@@ -0,0 +1,5 @@
+import { IndexStreamArgs } from '../database-index-api.js';
+export declare function readCsvFixture(filepath: URL): Promise<(IndexStreamArgs & {
+    createdAt?: Date | null | undefined;
+})[]>;
+//# sourceMappingURL=read-csv-fixture.util.d.ts.map

package/lib/__tests__/read-csv-fixture.util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-csv-fixture.util.d.ts","sourceRoot":"","sources":["../../src/__tests__/read-csv-fixture.util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAU1D,wBAAgB,cAAc,CAAC,QAAQ,EAAE,GAAG;;MA2B3C"}

package/lib/__tests__/read-csv-fixture.util.js

@@ -0,0 +1,35 @@
+import fs from 'node:fs';
+import csv from 'csv-parser';
+import { StreamID } from '@ceramicnetwork/streamid';
+import { CID } from 'multiformats/cid';
+const MODEL_ID = 'kjzl6cwe1jw145m7jxh4jpa6iw1ps3jcjordpo81e0w04krcpz8knxvg5ygiabd';
+const FAKE_CID = CID.parse('bafybeig6xv5nwphfmvcnektpnojts33jqcuam7bmye2pb54adnrtccjlsu');
+const MODEL = StreamID.fromString(MODEL_ID);
+export function readCsvFixture(filepath) {
+    return new Promise((resolve, reject) => {
+        const result = new Array();
+        const csvReader = csv({
+            separator: ';',
+            mapHeaders: ({ header }) => (header ? header.replace(/\s+/g, '') : null),
+        });
+        fs.createReadStream(filepath)
+            .pipe(csvReader)
+            .on('data', (row) => {
+            result.push({
+                model: MODEL,
+                streamID: StreamID.fromString(row.stream_id),
+                controller: row.controller,
+                streamContent: row.stream_content ? row.stream_content : '{}',
+                tip: row.tip ? row.tip : FAKE_CID,
+                lastAnchor: row.last_anchored_at ? new Date(Number(row.last_anchored_at) * 1000) : null,
+                createdAt: row.created_at ? new Date(Number(row.created_at) * 1000) : null,
+                firstAnchor: null,
+            });
+        })
+            .on('error', (error) => reject(error))
+            .on('end', () => {
+            resolve(result);
+        });
+    });
+}
+//# sourceMappingURL=read-csv-fixture.util.js.map

package/lib/__tests__/read-csv-fixture.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-csv-fixture.util.js","sourceRoot":"","sources":["../../src/__tests__/read-csv-fixture.util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,GAAG,MAAM,YAAY,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAEtC,MAAM,QAAQ,GAAG,iEAAiE,CAAA;AAClF,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAA;AACzF,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;AAE3C,MAAM,UAAU,cAAc,CAAC,QAAa;IAE1C,OAAO,IAAI,OAAO,CAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxD,MAAM,MAAM,GAAG,IAAI,KAAK,EAAc,CAAA;QACtC,MAAM,SAAS,GAAG,GAAG,CAAC;YACpB,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;SACzE,CAAC,CAAA;QACF,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC;aAC1B,IAAI,CAAC,SAAS,CAAC;aACf,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YAClB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;gBAC5C,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,aAAa,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;gBAC7D,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ;gBACjC,UAAU,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;gBACvF,SAAS,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC1E,WAAW,EAAE,IAAI;aAClB,CAAC,CAAA;QACJ,CAAC,CAAC;aACD,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACrC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACd,OAAO,CAAC,MAAM,CAAC,CAAA;QACjB,CAAC,CAAC,CAAA;IACN,CAAC,CAAC,CAAA;AACJ,CAAC"}

package/lib/as-table-name.util.d.ts

@@ -0,0 +1,3 @@
+import type { StreamID } from '@ceramicnetwork/streamid';
+export declare function asTableName(model: StreamID | string): string;
+//# sourceMappingURL=as-table-name.util.d.ts.map

package/lib/as-table-name.util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"as-table-name.util.d.ts","sourceRoot":"","sources":["../src/as-table-name.util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAKxD,wBAAgB,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG,MAAM,CAE5D"}

package/lib/as-table-name.util.js

@@ -0,0 +1,4 @@
+export function asTableName(model) {
+    return model.toString();
+}
+//# sourceMappingURL=as-table-name.util.js.map

package/lib/as-table-name.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"as-table-name.util.js","sourceRoot":"","sources":["../src/as-table-name.util.ts"],"names":[],"mappings":"AAKA,MAAM,UAAU,WAAW,CAAC,KAAwB;IAClD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAA;AACzB,CAAC"}

package/lib/build-indexing.d.ts

@@ -0,0 +1,13 @@
+import { type DatabaseIndexApi } from './database-index-api.js';
+import { DiagnosticsLogger, Networks } from '@ceramicnetwork/common';
+export declare type IndexingConfig = {
+    db: string;
+    allowQueriesBeforeHistoricalSync: boolean;
+    disableComposedb: boolean;
+    enableHistoricalSync: boolean;
+};
+export declare class UnsupportedDatabaseProtocolError extends Error {
+    constructor(protocol: string);
+}
+export declare function buildIndexing(indexingConfig: IndexingConfig, logger: DiagnosticsLogger, network: Networks): DatabaseIndexApi<Date | number>;
+//# sourceMappingURL=build-indexing.d.ts.map

package/lib/build-indexing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-indexing.d.ts","sourceRoot":"","sources":["../src/build-indexing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAAoC,MAAM,yBAAyB,CAAA;AAEjG,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAGpE,oBAAY,cAAc,GAAG;IAI3B,EAAE,EAAE,MAAM,CAAA;IAKV,gCAAgC,EAAE,OAAO,CAAA;IAKzC,gBAAgB,EAAE,OAAO,CAAA;IAKzB,oBAAoB,EAAE,OAAO,CAAA;CAC9B,CAAA;AAED,qBAAa,gCAAiC,SAAQ,KAAK;gBAC7C,QAAQ,EAAE,MAAM;CAG7B;AAmBD,wBAAgB,aAAa,CAC3B,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,QAAQ,GAChB,gBAAgB,CAAC,IAAI,GAAG,MAAM,CAAC,CAgDjC"}

package/lib/build-indexing.js

@@ -0,0 +1,55 @@
+import { PostgresIndexApi, SqliteIndexApi } from './database-index-api.js';
+import knex from 'knex';
+import * as fs from 'fs';
+export class UnsupportedDatabaseProtocolError extends Error {
+    constructor(protocol) {
+        super(`Not supported database protocol ${protocol}`);
+    }
+}
+class InvalidConnectionStringError extends Error {
+    constructor(connectionString) {
+        super(`Invalid database connection string: ${connectionString}`);
+    }
+}
+function parseURL(input) {
+    try {
+        return new URL(input);
+    }
+    catch {
+        throw new InvalidConnectionStringError(input);
+    }
+}
+export function buildIndexing(indexingConfig, logger, network) {
+    const connectionString = parseURL(indexingConfig.db);
+    const protocol = connectionString.protocol.replace(/:$/, '');
+    switch (protocol) {
+        case 'sqlite':
+        case 'sqlite3': {
+            logger.imp('Initializing SQLite connection');
+            if (fs) {
+                fs.mkdirSync(connectionString.pathname.substring(0, connectionString.pathname.lastIndexOf('/')), { recursive: true });
+            }
+            const dbConnection = knex({
+                client: 'sqlite3',
+                useNullAsDefault: true,
+                connection: {
+                    filename: connectionString.pathname,
+                },
+            });
+            return new SqliteIndexApi(dbConnection, indexingConfig.allowQueriesBeforeHistoricalSync, logger, network);
+        }
+        case 'postgres':
+        case 'postgresql': {
+            logger.imp('Initializing PostgreSQL connection');
+            const dataSource = knex({
+                client: 'pg',
+                connection: connectionString.toString(),
+                pool: { min: 0 },
+            });
+            return new PostgresIndexApi(dataSource, indexingConfig.allowQueriesBeforeHistoricalSync, logger, network);
+        }
+        default:
+            throw new UnsupportedDatabaseProtocolError(protocol);
+    }
+}
+//# sourceMappingURL=build-indexing.js.map

package/lib/build-indexing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-indexing.js","sourceRoot":"","sources":["../src/build-indexing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,gBAAgB,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACjG,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAA;AAwBxB,MAAM,OAAO,gCAAiC,SAAQ,KAAK;IACzD,YAAY,QAAgB;QAC1B,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAA;IACtD,CAAC;CACF;AAED,MAAM,4BAA6B,SAAQ,KAAK;IAC9C,YAAY,gBAAwB;QAClC,KAAK,CAAC,uCAAuC,gBAAgB,EAAE,CAAC,CAAA;IAClE,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI;QACF,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;KACtB;IAAC,MAAM;QACN,MAAM,IAAI,4BAA4B,CAAC,KAAK,CAAC,CAAA;KAC9C;AACH,CAAC;AAKD,MAAM,UAAU,aAAa,CAC3B,cAA8B,EAC9B,MAAyB,EACzB,OAAiB;IAEjB,MAAM,gBAAgB,GAAG,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;IACpD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAC5D,QAAQ,QAAQ,EAAE;QAChB,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAA;YAC5C,IAAI,EAAE,EAAE;gBAIN,EAAE,CAAC,SAAS,CACV,gBAAgB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAClF,EAAE,SAAS,EAAE,IAAI,EAAE,CACpB,CAAA;aACF;YACD,MAAM,YAAY,GAAG,IAAI,CAAC;gBACxB,MAAM,EAAE,SAAS;gBACjB,gBAAgB,EAAE,IAAI;gBACtB,UAAU,EAAE;oBACV,QAAQ,EAAE,gBAAgB,CAAC,QAAQ;iBACpC;aACF,CAAC,CAAA;YACF,OAAO,IAAI,cAAc,CACvB,YAAY,EACZ,cAAc,CAAC,gCAAgC,EAC/C,MAAM,EACN,OAAO,CACR,CAAA;SACF;QACD,KAAK,UAAU,CAAC;QAChB,KAAK,YAAY,CAAC,CAAC;YACjB,MAAM,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAA;YAChD,MAAM,UAAU,GAAG,IAAI,CAAC;gBACtB,MAAM,EAAE,IAAI;gBACZ,UAAU,EAAE,gBAAgB,CAAC,QAAQ,EAAE;gBACvC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;aACjB,CAAC,CAAA;YACF,OAAO,IAAI,gBAAgB,CACzB,UAAU,EACV,cAAc,CAAC,gCAAgC,EAC/C,MAAM,EACN,OAAO,CACR,CAAA;SACF;QACD;YACE,MAAM,IAAI,gCAAgC,CAAC,QAAQ,CAAC,CAAA;KACvD;AACH,CAAC"}

package/lib/column-name.util.d.ts

@@ -0,0 +1,2 @@
+export declare function addColumnPrefix(name: string): string;
+//# sourceMappingURL=column-name.util.d.ts.map

package/lib/column-name.util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"column-name.util.d.ts","sourceRoot":"","sources":["../src/column-name.util.ts"],"names":[],"mappings":"AAEA,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpD"}

package/lib/column-name.util.js

@@ -0,0 +1,5 @@
+const CUSTOM_COLUMN_PREFIX = 'custom_';
+export function addColumnPrefix(name) {
+    return name.startsWith(CUSTOM_COLUMN_PREFIX) ? name : CUSTOM_COLUMN_PREFIX + name;
+}
+//# sourceMappingURL=column-name.util.js.map

package/lib/column-name.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"column-name.util.js","sourceRoot":"","sources":["../src/column-name.util.ts"],"names":[],"mappings":"AAAA,MAAM,oBAAoB,GAAG,SAAS,CAAA;AAEtC,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,OAAO,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,oBAAoB,GAAG,IAAI,CAAA;AACnF,CAAC"}

package/lib/config.d.ts

@@ -0,0 +1,2 @@
+export declare const CONFIG_TABLE_NAME = "ceramic_config";
+//# sourceMappingURL=config.d.ts.map

package/lib/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,iBAAiB,mBAAmB,CAAA"}

package/lib/config.js

@@ -0,0 +1,2 @@
+export const CONFIG_TABLE_NAME = 'ceramic_config';
+//# sourceMappingURL=config.js.map

package/lib/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAA"}

package/lib/database-index-api.d.ts

@@ -0,0 +1,94 @@
+import { StreamID } from '@ceramicnetwork/streamid';
+import type { BaseQuery, Pagination, Page, DiagnosticsLogger, Networks, FieldsIndex, ModelData } from '@ceramicnetwork/common';
+import { Knex } from 'knex';
+import type { CID } from 'multiformats/cid';
+import { ModelRelationsDefinition } from '@ceramicnetwork/stream-model';
+import { TablesManager } from './tables-manager.js';
+import { ISyncQueryApi } from './history-sync/interfaces.js';
+export declare const INDEXED_MODEL_CONFIG_TABLE_NAME = "ceramic_models";
+export interface IndexStreamArgs {
+    readonly streamID: StreamID;
+    readonly model: StreamID;
+    readonly controller: string;
+    readonly streamContent: Record<string, any>;
+    readonly tip: CID;
+    readonly lastAnchor: Date | null;
+    readonly firstAnchor: Date | null;
+}
+export declare function fieldsIndexName(idx: FieldsIndex, table: string): string;
+export interface IndexModelArgs {
+    readonly model: StreamID;
+    relations?: ModelRelationsDefinition;
+    indices?: Array<FieldsIndex>;
+}
+declare type IndexedData<DateType> = {
+    stream_id: string;
+    controller_did: string;
+    stream_content: Record<string, any> | string;
+    tip: string;
+    last_anchored_at: DateType | null;
+    first_anchored_at: DateType | null;
+    created_at?: DateType;
+    updated_at: DateType;
+};
+export declare abstract class DatabaseIndexApi<DateType = Date | number> {
+    protected readonly dbConnection: Knex;
+    private readonly allowQueriesBeforeHistoricalSync;
+    private readonly logger;
+    private readonly network;
+    private readonly insertionOrder;
+    private indexedModels;
+    private readonly modelRelations;
+    tablesManager: TablesManager;
+    syncApi: ISyncQueryApi;
+    protected constructor(dbConnection: Knex, allowQueriesBeforeHistoricalSync: boolean, logger: DiagnosticsLogger, network: Networks);
+    abstract getIndexedData(indexingArgs: IndexStreamArgs & {
+        createdAt?: Date;
+        updatedAt?: Date;
+    }): IndexedData<DateType>;
+    abstract now(): DateType;
+    abstract parseIndices(indices: unknown): Array<FieldsIndex>;
+    setSyncQueryApi(api: ISyncQueryApi): void;
+    indexModels(models: Array<IndexModelArgs>): Promise<void>;
+    private indexModelsInDatabase;
+    stopIndexingModels(models: Array<ModelData>): Promise<void>;
+    private stopIndexingModelsInDatabase;
+    indexStream(indexingArgs: IndexStreamArgs & {
+        createdAt?: Date;
+        updatedAt?: Date;
+    }): Promise<void>;
+    getIndexedModels(): Array<ModelData>;
+    private getIndexedModelsFromDatabase;
+    getModelsNoLongerIndexed(): Promise<Array<ModelData>>;
+    assertModelQueryable(modelStreamId: StreamID): Promise<void>;
+    assertModelIsIndexed(modelStreamId: StreamID): Promise<void>;
+    assertNoOngoingSyncForModel(modelStreamId: StreamID): Promise<void>;
+    abstract getCountFromResult(response: Array<Record<string, string | number>>): number;
+    count(query: BaseQuery): Promise<number>;
+    page(query: BaseQuery & Pagination): Promise<Page<StreamID>>;
+    init(): Promise<void>;
+    close(): Promise<void>;
+}
+export declare class PostgresIndexApi extends DatabaseIndexApi<Date> {
+    constructor(dbConnection: Knex, allowQueriesBeforeHistoricalSync: boolean, logger: DiagnosticsLogger, network: Networks);
+    now(): Date;
+    getCountFromResult(response: Array<Record<string, string | number>>): number;
+    getIndexedData(indexingArgs: IndexStreamArgs & {
+        createdAt?: Date;
+        updatedAt?: Date;
+    }): IndexedData<Date>;
+    parseIndices(indices: Array<FieldsIndex>): Array<FieldsIndex>;
+}
+export declare function asTimestamp(input: Date | null | undefined): number | null;
+export declare class SqliteIndexApi extends DatabaseIndexApi<number> {
+    constructor(dbConnection: Knex, allowQueriesBeforeHistoricalSync: boolean, logger: DiagnosticsLogger, network: Networks);
+    now(): number;
+    getCountFromResult(response: [Record<string, string | number>]): number;
+    getIndexedData(indexingArgs: IndexStreamArgs & {
+        createdAt?: Date;
+        updatedAt?: Date;
+    }): IndexedData<number>;
+    parseIndices(indices: string): Array<FieldsIndex>;
+}
+export {};
+//# sourceMappingURL=database-index-api.d.ts.map

package/lib/database-index-api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database-index-api.d.ts","sourceRoot":"","sources":["../src/database-index-api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AACnD,OAAO,KAAK,EACV,SAAS,EACT,UAAU,EACV,IAAI,EACJ,iBAAiB,EACjB,QAAQ,EACR,WAAW,EACX,SAAS,EACV,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAC3B,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAA;AAIvE,OAAO,EAAE,aAAa,EAA8C,MAAM,qBAAqB,CAAA;AAE/F,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAI5D,eAAO,MAAM,+BAA+B,mBAAmB,CAAA;AAE/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAA;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC3C,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAA;IACjB,QAAQ,CAAC,UAAU,EAAE,IAAI,GAAG,IAAI,CAAA;IAChC,QAAQ,CAAC,WAAW,EAAE,IAAI,GAAG,IAAI,CAAA;CAClC;AAOD,wBAAgB,eAAe,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAMvE;AAQD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAA;IACxB,SAAS,CAAC,EAAE,wBAAwB,CAAA;IACpC,OAAO,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;CAC7B;AAED,aAAK,WAAW,CAAC,QAAQ,IAAI;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAA;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,gBAAgB,EAAE,QAAQ,GAAG,IAAI,CAAA;IACjC,iBAAiB,EAAE,QAAQ,GAAG,IAAI,CAAA;IAClC,UAAU,CAAC,EAAE,QAAQ,CAAA;IACrB,UAAU,EAAE,QAAQ,CAAA;CACrB,CAAA;AAKD,8BAAsB,gBAAgB,CAAC,QAAQ,GAAG,IAAI,GAAG,MAAM;IAU3D,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,IAAI;IACrC,OAAO,CAAC,QAAQ,CAAC,gCAAgC;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAZ1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAC/C,OAAO,CAAC,aAAa,CAAuB;IAG5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAmC;IAClE,aAAa,EAAG,aAAa,CAAA;IAC7B,OAAO,EAAG,aAAa,CAAA;IAEvB,SAAS,aACY,YAAY,EAAE,IAAI,EACpB,gCAAgC,EAAE,OAAO,EACzC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,QAAQ;IAKpC,QAAQ,CAAC,cAAc,CACrB,YAAY,EAAE,eAAe,GAAG;QAAE,SAAS,CAAC,EAAE,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,IAAI,CAAA;KAAE,GACrE,WAAW,CAAC,QAAQ,CAAC;IACxB,QAAQ,CAAC,GAAG,IAAI,QAAQ;IACxB,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC;IAE3D,eAAe,CAAC,GAAG,EAAE,aAAa;IAS5B,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAmBjD,qBAAqB;IA+B7B,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;YAOnD,4BAA4B;IA2BpC,WAAW,CACf,YAAY,EAAE,eAAe,GAAG;QAAE,SAAS,CAAC,EAAE,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,IAAI,CAAA;KAAE,GACrE,OAAO,CAAC,IAAI,CAAC;IAeT,gBAAgB,IAAI,KAAK,CAAC,SAAS,CAAC;YAS7B,4BAA4B;IAapC,wBAAwB,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAgBrD,oBAAoB,CAAC,aAAa,EAAE,QAAQ;IAS5C,oBAAoB,CAAC,aAAa,EAAE,QAAQ;IAiB5C,2BAA2B,CAAC,aAAa,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,MAAM;IAK/E,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IAexC,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAS5D,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED,qBAAa,gBAAiB,SAAQ,gBAAgB,CAAC,IAAI,CAAC;gBAExD,YAAY,EAAE,IAAI,EAClB,gCAAgC,EAAE,OAAO,EACzC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,QAAQ;IAMnB,GAAG,IAAI,IAAI;IAKX,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,MAAM;IAI5E,cAAc,CACZ,YAAY,EAAE,eAAe,GAAG;QAAE,SAAS,CAAC,EAAE,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,IAAI,CAAA;KAAE,GACrE,WAAW,CAAC,IAAI,CAAC;IAcpB,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC;CAG9D;AAKD,wBAAgB,WAAW,CAAC,KAAK,EAAE,IAAI,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAMzE;AAED,qBAAa,cAAe,SAAQ,gBAAgB,CAAC,MAAM,CAAC;gBAExD,YAAY,EAAE,IAAI,EAClB,gCAAgC,EAAE,OAAO,EACzC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,QAAQ;IAMnB,GAAG,IAAI,MAAM;IAIb,kBAAkB,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,MAAM;IAIvE,cAAc,CACZ,YAAY,EAAE,eAAe,GAAG;QAAE,SAAS,CAAC,EAAE,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,IAAI,CAAA;KAAE,GACrE,WAAW,CAAC,MAAM,CAAC;IActB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC;CAGlD"}

package/lib/database-index-api.js

@@ -0,0 +1,228 @@
+import { StreamID } from '@ceramicnetwork/streamid';
+import { InsertionOrder } from './insertion-order.js';
+import { asTableName } from './as-table-name.util.js';
+import { IndexQueryNotAvailableError } from './index-query-not-available.error.js';
+import { PostgresTablesManager, SqliteTablesManager } from './tables-manager.js';
+import { addColumnPrefix } from './column-name.util.js';
+import cloneDeep from 'lodash.clonedeep';
+import { indexNameFromTableName } from './migrations/1-create-model-table.js';
+export const INDEXED_MODEL_CONFIG_TABLE_NAME = 'ceramic_models';
+export function fieldsIndexName(idx, table) {
+    const fieldPath = idx.fields
+        .flatMap((f) => f.path)
+        .map((p) => p.slice(0, 5))
+        .join('_');
+    return `${indexNameFromTableName(table)}_${fieldPath}`.slice(0, 64);
+}
+export class DatabaseIndexApi {
+    constructor(dbConnection, allowQueriesBeforeHistoricalSync, logger, network) {
+        this.dbConnection = dbConnection;
+        this.allowQueriesBeforeHistoricalSync = allowQueriesBeforeHistoricalSync;
+        this.logger = logger;
+        this.network = network;
+        this.indexedModels = [];
+        this.modelRelations = new Map();
+        this.insertionOrder = new InsertionOrder(dbConnection);
+    }
+    setSyncQueryApi(api) {
+        this.syncApi = api;
+    }
+    async indexModels(models) {
+        await this.indexModelsInDatabase(models);
+        for (const modelArgs of models) {
+            await this.assertNoOngoingSyncForModel(modelArgs.model);
+            const foundModelToIndex = this.indexedModels.find((indexedModel) => indexedModel.streamID.equals(modelArgs.model));
+            if (!foundModelToIndex) {
+                this.indexedModels.push({
+                    streamID: modelArgs.model,
+                    indices: modelArgs.indices,
+                });
+            }
+            if (modelArgs.relations) {
+                this.modelRelations.set(modelArgs.model.toString(), Object.keys(modelArgs.relations));
+            }
+        }
+    }
+    async indexModelsInDatabase(models) {
+        if (models.length === 0)
+            return;
+        await this.tablesManager.initMidTables(models);
+        await this.tablesManager.verifyTables(models);
+        await this.dbConnection(INDEXED_MODEL_CONFIG_TABLE_NAME)
+            .insert(models.map((indexModelArgs) => {
+            return {
+                model: indexModelArgs.model.toString(),
+                ...(indexModelArgs.indices && { indices: JSON.stringify(indexModelArgs.indices) }),
+                is_indexed: true,
+                updated_by: '0',
+                updated_at: this.now(),
+            };
+        }))
+            .onConflict('model')
+            .merge({
+            updated_at: this.now(),
+            is_indexed: true,
+            updated_by: '0',
+        });
+    }
+    async stopIndexingModels(models) {
+        await this.stopIndexingModelsInDatabase(models);
+        this.indexedModels = this.indexedModels.filter((idx) => !models.some((data) => data.streamID.equals(idx.streamID)));
+    }
+    async stopIndexingModelsInDatabase(models) {
+        if (models.length === 0)
+            return;
+        await this.dbConnection(INDEXED_MODEL_CONFIG_TABLE_NAME)
+            .insert(models.map((model) => {
+            return {
+                model: model.streamID.toString(),
+                is_indexed: false,
+                updated_by: '0',
+                updated_at: this.now(),
+            };
+        }))
+            .onConflict('model')
+            .merge({
+            updated_at: this.now(),
+            is_indexed: false,
+            updated_by: '0',
+        });
+    }
+    async indexStream(indexingArgs) {
+        const tableName = asTableName(indexingArgs.model);
+        const indexedData = this.getIndexedData(indexingArgs);
+        const relations = this.modelRelations.get(indexingArgs.model.toString()) ?? [];
+        for (const relation of relations) {
+            indexedData[addColumnPrefix(relation)] = indexingArgs.streamContent[relation];
+        }
+        const toMerge = cloneDeep(indexedData);
+        delete toMerge['created_at'];
+        await this.dbConnection(tableName).insert(indexedData).onConflict('stream_id').merge(toMerge);
+    }
+    getIndexedModels() {
+        return this.indexedModels;
+    }
+    async getIndexedModelsFromDatabase() {
+        return (await this.dbConnection(INDEXED_MODEL_CONFIG_TABLE_NAME).select('model', 'indices').where({
+            is_indexed: true,
+        })).map((result) => {
+            return {
+                streamID: StreamID.fromString(result.model),
+                indices: this.parseIndices(result.indices),
+            };
+        });
+    }
+    async getModelsNoLongerIndexed() {
+        return (await this.dbConnection(INDEXED_MODEL_CONFIG_TABLE_NAME).select('model', 'indices').where({
+            is_indexed: false,
+        })).map((result) => {
+            return {
+                streamID: StreamID.fromString(result.model),
+                indices: result.indices,
+            };
+        });
+    }
+    async assertModelQueryable(modelStreamId) {
+        await this.assertModelIsIndexed(modelStreamId);
+        await this.assertNoOngoingSyncForModel(modelStreamId);
+    }
+    async assertModelIsIndexed(modelStreamId) {
+        const foundModelToIndex = this.indexedModels.find((indexedModel) => modelStreamId.equals(indexedModel.streamID));
+        if (foundModelToIndex == undefined) {
+            const err = new Error(`Query failed: Model ${modelStreamId.toString()} is not indexed on this node`);
+            this.logger.debug(err);
+            throw err;
+        }
+    }
+    async assertNoOngoingSyncForModel(modelStreamId) {
+        if (!this.allowQueriesBeforeHistoricalSync &&
+            !(await this.syncApi.syncComplete(modelStreamId.toString()))) {
+            throw new IndexQueryNotAvailableError(modelStreamId);
+        }
+    }
+    async count(query) {
+        const model = StreamID.fromString(query.model.toString());
+        await this.assertModelQueryable(model);
+        const tableName = asTableName(query.model);
+        let dbQuery = this.dbConnection(tableName).count('*');
+        dbQuery = this.insertionOrder.applyFilters(dbQuery, query);
+        return dbQuery.then((response) => this.getCountFromResult(response));
+    }
+    async page(query) {
+        const model = StreamID.fromString(query.model.toString());
+        await this.assertModelQueryable(model);
+        return this.insertionOrder.page(query);
+    }
+    async init() {
+        await this.tablesManager.initConfigTables(this.network);
+        this.indexedModels = await this.getIndexedModelsFromDatabase();
+    }
+    async close() {
+        await this.dbConnection.destroy();
+    }
+}
+export class PostgresIndexApi extends DatabaseIndexApi {
+    constructor(dbConnection, allowQueriesBeforeHistoricalSync, logger, network) {
+        super(dbConnection, allowQueriesBeforeHistoricalSync, logger, network);
+        this.tablesManager = new PostgresTablesManager(dbConnection, logger);
+    }
+    now() {
+        return new Date();
+    }
+    getCountFromResult(response) {
+        return Number(response[0]['count']);
+    }
+    getIndexedData(indexingArgs) {
+        const now = this.now();
+        return {
+            stream_id: indexingArgs.streamID.toString(),
+            controller_did: indexingArgs.controller.toString(),
+            stream_content: indexingArgs.streamContent,
+            tip: indexingArgs.tip.toString(),
+            last_anchored_at: indexingArgs.lastAnchor,
+            first_anchored_at: indexingArgs.firstAnchor,
+            created_at: indexingArgs.createdAt || now,
+            updated_at: indexingArgs.updatedAt || now,
+        };
+    }
+    parseIndices(indices) {
+        return indices ?? undefined;
+    }
+}
+export function asTimestamp(input) {
+    if (input) {
+        return input.valueOf();
+    }
+    else {
+        return null;
+    }
+}
+export class SqliteIndexApi extends DatabaseIndexApi {
+    constructor(dbConnection, allowQueriesBeforeHistoricalSync, logger, network) {
+        super(dbConnection, allowQueriesBeforeHistoricalSync, logger, network);
+        this.tablesManager = new SqliteTablesManager(dbConnection, logger);
+    }
+    now() {
+        return new Date().valueOf();
+    }
+    getCountFromResult(response) {
+        return Number(response[0]['count(*)']);
+    }
+    getIndexedData(indexingArgs) {
+        const now = this.now();
+        return {
+            stream_id: indexingArgs.streamID.toString(),
+            controller_did: indexingArgs.controller.toString(),
+            stream_content: JSON.stringify(indexingArgs.streamContent),
+            tip: indexingArgs.tip.toString(),
+            last_anchored_at: asTimestamp(indexingArgs.lastAnchor),
+            first_anchored_at: asTimestamp(indexingArgs.firstAnchor),
+            created_at: asTimestamp(indexingArgs.createdAt) || now,
+            updated_at: asTimestamp(indexingArgs.updatedAt) || now,
+        };
+    }
+    parseIndices(indices) {
+        return indices ? JSON.parse(indices) : undefined;
+    }
+}
+//# sourceMappingURL=database-index-api.js.map

package/lib/database-index-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database-index-api.js","sourceRoot":"","sources":["../src/database-index-api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAanD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAA;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,sCAAsC,CAAA;AAClF,OAAO,EAAiB,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAC/F,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAEvD,OAAO,SAAS,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAA;AAE7E,MAAM,CAAC,MAAM,+BAA+B,GAAG,gBAAgB,CAAA;AAiB/D,MAAM,UAAU,eAAe,CAAC,GAAgB,EAAE,KAAa;IAC7D,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM;SACzB,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACzB,IAAI,CAAC,GAAG,CAAC,CAAA;IACZ,OAAO,GAAG,sBAAsB,CAAC,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AACrE,CAAC;AA4BD,MAAM,OAAgB,gBAAgB;IASpC,YACqB,YAAkB,EACpB,gCAAyC,EACzC,MAAyB,EACzB,OAAiB;QAHf,iBAAY,GAAZ,YAAY,CAAM;QACpB,qCAAgC,GAAhC,gCAAgC,CAAS;QACzC,WAAM,GAAN,MAAM,CAAmB;QACzB,YAAO,GAAP,OAAO,CAAU;QAX5B,kBAAa,GAAqB,EAAE,CAAA;QAG3B,mBAAc,GAAG,IAAI,GAAG,EAAyB,CAAA;QAUhE,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAA;IACxD,CAAC;IAQD,eAAe,CAAC,GAAkB;QAChC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAA;IACpB,CAAC;IAOD,KAAK,CAAC,WAAW,CAAC,MAA6B;QAC7C,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAA;QACxC,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE;YAC9B,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YACvD,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACjE,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAC9C,CAAA;YACD,IAAI,CAAC,iBAAiB,EAAE;gBACtB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;oBACtB,QAAQ,EAAE,SAAS,CAAC,KAAK;oBACzB,OAAO,EAAE,SAAS,CAAC,OAAO;iBAC3B,CAAC,CAAA;aACH;YACD,IAAI,SAAS,CAAC,SAAS,EAAE;gBACvB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAA;aACtF;SACF;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,MAA6B;QAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAC/B,MAAM,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;QAG7C,MAAM,IAAI,CAAC,YAAY,CAAC,+BAA+B,CAAC;aACrD,MAAM,CACL,MAAM,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;YAC5B,OAAO;gBACL,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE;gBACtC,GAAG,CAAC,cAAc,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClF,UAAU,EAAE,IAAI;gBAChB,UAAU,EAAE,GAAG;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;aACvB,CAAA;QACH,CAAC,CAAC,CACH;aACA,UAAU,CAAC,OAAO,CAAC;aACnB,KAAK,CAAC;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;YACtB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,GAAG;SAChB,CAAC,CAAA;IACN,CAAC;IAOD,KAAK,CAAC,kBAAkB,CAAC,MAAwB;QAC/C,MAAM,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAA;QAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAC5C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CACpE,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,4BAA4B,CAAC,MAAwB;QACjE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE/B,MAAM,IAAI,CAAC,YAAY,CAAC,+BAA+B,CAAC;aACrD,MAAM,CACL,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACnB,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAChC,UAAU,EAAE,KAAK;gBACjB,UAAU,EAAE,GAAG;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;aACvB,CAAA;QACH,CAAC,CAAC,CACH;aACA,UAAU,CAAC,OAAO,CAAC;aACnB,KAAK,CAAC;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;YACtB,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,GAAG;SAChB,CAAC,CAAA;IACN,CAAC;IAOD,KAAK,CAAC,WAAW,CACf,YAAsE;QAEtE,MAAM,SAAS,GAAG,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,CAA4B,CAAA;QAChF,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;QAC9E,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,WAAW,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,GAAG,YAAY,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAA;SAC9E;QACD,MAAM,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,CAAA;QACtC,OAAO,OAAO,CAAC,YAAY,CAAC,CAAA;QAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC/F,CAAC;IAKM,gBAAgB;QAMrB,OAAO,IAAI,CAAC,aAAa,CAAA;IAC3B,CAAC;IAEO,KAAK,CAAC,4BAA4B;QACxC,OAAO,CACL,MAAM,IAAI,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC;YACxF,UAAU,EAAE,IAAI;SACjB,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,OAAO;gBACL,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC3C,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC;aAC3C,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,wBAAwB;QAC5B,OAAO,CACL,MAAM,IAAI,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC;YACxF,UAAU,EAAE,KAAK;SAClB,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,OAAO;gBACL,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC3C,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAKD,KAAK,CAAC,oBAAoB,CAAC,aAAuB;QAChD,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAA;QAC9C,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAA;IACvD,CAAC;IAMD,KAAK,CAAC,oBAAoB,CAAC,aAAuB;QAChD,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CACjE,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAC5C,CAAA;QACD,IAAI,iBAAiB,IAAI,SAAS,EAAE;YAClC,MAAM,GAAG,GAAG,IAAI,KAAK,CACnB,uBAAuB,aAAa,CAAC,QAAQ,EAAE,8BAA8B,CAC9E,CAAA;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YACtB,MAAM,GAAG,CAAA;SACV;IACH,CAAC;IAMD,KAAK,CAAC,2BAA2B,CAAC,aAAuB;QACvD,IACE,CAAC,IAAI,CAAC,gCAAgC;YACtC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC,EAC5D;YACA,MAAM,IAAI,2BAA2B,CAAC,aAAa,CAAC,CAAA;SACrD;IACH,CAAC;IAOD,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAA;QACzD,MAAM,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;QAEtC,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC1C,IAAI,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAErD,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAE1D,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAA;IACtE,CAAC;IAKD,KAAK,CAAC,IAAI,CAAC,KAA6B;QACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAA;QACzD,MAAM,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;QACtC,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACxC,CAAC;IAKD,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACvD,IAAI,CAAC,aAAa,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAA;IAChE,CAAC;IAKD,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAA;IACnC,CAAC;CACF;AAED,MAAM,OAAO,gBAAiB,SAAQ,gBAAsB;IAC1D,YACE,YAAkB,EAClB,gCAAyC,EACzC,MAAyB,EACzB,OAAiB;QAEjB,KAAK,CAAC,YAAY,EAAE,gCAAgC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;QACtE,IAAI,CAAC,aAAa,GAAG,IAAI,qBAAqB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACtE,CAAC;IAED,GAAG;QAED,OAAO,IAAI,IAAI,EAAE,CAAA;IACnB,CAAC;IAED,kBAAkB,CAAC,QAAgD;QACjE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,CAAA;IACtC,CAAC;IAED,cAAc,CACZ,YAAsE;QAEtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,OAAO;YACL,SAAS,EAAE,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE;YAC3C,cAAc,EAAE,YAAY,CAAC,UAAU,CAAC,QAAQ,EAAE;YAClD,cAAc,EAAE,YAAY,CAAC,aAAa;YAC1C,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAChC,gBAAgB,EAAE,YAAY,CAAC,UAAU;YACzC,iBAAiB,EAAE,YAAY,CAAC,WAAW;YAC3C,UAAU,EAAE,YAAY,CAAC,SAAS,IAAI,GAAG;YACzC,UAAU,EAAE,YAAY,CAAC,SAAS,IAAI,GAAG;SAC1C,CAAA;IACH,CAAC;IAED,YAAY,CAAC,OAA2B;QACtC,OAAO,OAAO,IAAI,SAAS,CAAA;IAC7B,CAAC;CACF;AAKD,MAAM,UAAU,WAAW,CAAC,KAA8B;IACxD,IAAI,KAAK,EAAE;QACT,OAAO,KAAK,CAAC,OAAO,EAAE,CAAA;KACvB;SAAM;QACL,OAAO,IAAI,CAAA;KACZ;AACH,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,gBAAwB;IAC1D,YACE,YAAkB,EAClB,gCAAyC,EACzC,MAAyB,EACzB,OAAiB;QAEjB,KAAK,CAAC,YAAY,EAAE,gCAAgC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;QACtE,IAAI,CAAC,aAAa,GAAG,IAAI,mBAAmB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACpE,CAAC;IAED,GAAG;QACD,OAAO,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAA;IAC7B,CAAC;IAED,kBAAkB,CAAC,QAA2C;QAC5D,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAA;IACxC,CAAC;IAED,cAAc,CACZ,YAAsE;QAEtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,OAAO;YACL,SAAS,EAAE,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE;YAC3C,cAAc,EAAE,YAAY,CAAC,UAAU,CAAC,QAAQ,EAAE;YAClD,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,aAAa,CAAC;YAC1D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAChC,gBAAgB,EAAE,WAAW,CAAC,YAAY,CAAC,UAAU,CAAC;YACtD,iBAAiB,EAAE,WAAW,CAAC,YAAY,CAAC,WAAW,CAAC;YACxD,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,GAAG;YACtD,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,GAAG;SACvD,CAAA;IACH,CAAC;IAED,YAAY,CAAC,OAAe;QAC1B,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAClD,CAAC;CACF"}

package/lib/history-sync/interfaces.d.ts

@@ -0,0 +1,52 @@
+import type { CID } from 'multiformats/cid';
+import type { StreamID } from '@ceramicnetwork/streamid';
+import type { SupportedNetwork } from '@ceramicnetwork/anchor-utils';
+export declare type SyncConfig = {
+    db: string;
+    chainId: SupportedNetwork;
+};
+export interface ISyncQueryApi {
+    syncComplete(model: string): boolean;
+}
+export interface ISyncApi extends ISyncQueryApi {
+    startModelSync(models: string | string[], startBlock?: number, endBlock?: number): Promise<void>;
+    stopModelSync(models: string | string[]): Promise<void>;
+    shutdown(): Promise<void>;
+}
+export declare type HandleCommit = (streamId: StreamID, commit: CID, model?: StreamID) => Promise<void>;
+export interface IpfsService {
+    retrieveFromIPFS(cid: CID | string, path?: string): Promise<any>;
+    retrieveCommit(cid: CID | string, streamId: StreamID): Promise<any>;
+    storeCommit(data: any, streamId?: StreamID): Promise<CID>;
+    storeRecord(record: Record<string, unknown>): Promise<CID>;
+}
+export interface TreeMetadata {
+    numEntries: number;
+    streamIds: string[];
+}
+export declare const REBUILD_ANCHOR_JOB = "rebuildAnchorJob";
+export interface RebuildAnchorJobData {
+    models: string[];
+    chainId: string;
+    txHash: string;
+    root: string;
+    txType?: string;
+}
+export declare const HISTORY_SYNC_JOB = "historySyncJob";
+export declare const CONTINUOUS_SYNC_JOB = "continuousSyncJob";
+export declare type SyncJob = typeof HISTORY_SYNC_JOB | typeof CONTINUOUS_SYNC_JOB;
+export declare enum SyncJobType {
+    Catchup = 0,
+    Reorg = 1,
+    Full = 2,
+    Continuous = 3
+}
+export interface SyncJobData {
+    currentBlock?: number;
+    jobType: SyncJobType;
+    fromBlock: number;
+    toBlock: number;
+    models: string[];
+}
+export declare type JobData = RebuildAnchorJobData | SyncJobData;
+//# sourceMappingURL=interfaces.d.ts.map

package/lib/history-sync/interfaces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../src/history-sync/interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AACxD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAA;AAEpE,oBAAY,UAAU,GAAG;IAIvB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,gBAAgB,CAAA;CAC1B,CAAA;AAKD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAA;CACrC;AAMD,MAAM,WAAW,QAAS,SAAQ,aAAa;IAC7C,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChG,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACvD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1B;AAID,oBAAY,YAAY,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;AAE/F,MAAM,WAAW,WAAW;IAC1B,gBAAgB,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;IAChE,cAAc,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;IACnE,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;IACzD,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;CAC3D;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,EAAE,CAAA;CACpB;AAED,eAAO,MAAM,kBAAkB,qBAAqB,CAAA;AACpD,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,eAAO,MAAM,gBAAgB,mBAAmB,CAAA;AAChD,eAAO,MAAM,mBAAmB,sBAAsB,CAAA;AAEtD,oBAAY,OAAO,GAAG,OAAO,gBAAgB,GAAG,OAAO,mBAAmB,CAAA;AAC1E,oBAAY,WAAW;IACrB,OAAO,IAAA;IACP,KAAK,IAAA;IACL,IAAI,IAAA;IACJ,UAAU,IAAA;CACX;AACD,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,WAAW,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,oBAAY,OAAO,GAAG,oBAAoB,GAAG,WAAW,CAAA"}