@ceramicnetwork/indexing 1.1.0-rc.0

package/lib/history-sync/workers/sync.js

@@ -0,0 +1,77 @@
+import { createBlocksProofsLoader } from '@ceramicnetwork/anchor-listener';
+import { concatMap, lastValueFrom, of, catchError } from 'rxjs';
+import { createRebuildAnchorJob } from './rebuild-anchor.js';
+import { HISTORY_SYNC_JOB, CONTINUOUS_SYNC_JOB, } from '../interfaces.js';
+const SYNC_JOB_OPTIONS = {
+    retryLimit: 5,
+    retryDelay: 60,
+    retryBackoff: true,
+    expireInHours: 12,
+    retentionDays: 3,
+};
+export function createContinuousSyncJob(data, options) {
+    return {
+        name: CONTINUOUS_SYNC_JOB,
+        data,
+        options: options || SYNC_JOB_OPTIONS,
+    };
+}
+export function createHistorySyncJob(data, options) {
+    return {
+        name: HISTORY_SYNC_JOB,
+        data,
+        options: options || SYNC_JOB_OPTIONS,
+    };
+}
+export class SyncWorker {
+    constructor(provider, jobQueue, chainId, logger, syncCompleteCallback) {
+        this.provider = provider;
+        this.jobQueue = jobQueue;
+        this.chainId = chainId;
+        this.logger = logger;
+        this.syncCompleteCallback = syncCompleteCallback;
+    }
+    async handler(job) {
+        const jobData = job.data;
+        const { jobType, fromBlock, toBlock, models } = jobData;
+        const currentBlock = jobData.currentBlock || fromBlock;
+        const blockProof$ = createBlocksProofsLoader({
+            provider: this.provider,
+            chainId: this.chainId,
+            fromBlock: currentBlock,
+            toBlock,
+        }).pipe(catchError((err) => {
+            this.logger.err(`Received error when retrieving block proofs for models ${models} from block ${currentBlock} to block ${toBlock}: ${err}`);
+            return of(null);
+        }), concatMap(async (blockProofs) => {
+            if (!blockProofs) {
+                throw Error('Error loading block proof');
+            }
+            const { proofs, blockNumber } = blockProofs;
+            if (proofs.length > 0) {
+                const jobs = proofs.map((proof) => createRebuildAnchorJob(proof, models));
+                await this.jobQueue.addJobs(jobs);
+                this.logger.debug(`Successfully created ${jobs.length} rebuild anchor commit jobs for block ${blockNumber}`);
+            }
+            await this.jobQueue.updateJob(job.id, {
+                fromBlock,
+                currentBlock: blockNumber + 1,
+                toBlock,
+                models,
+                jobType,
+            });
+        }));
+        await lastValueFrom(blockProof$).then(() => {
+            if (this.syncCompleteCallback) {
+                for (const model of models) {
+                    this.syncCompleteCallback({
+                        jobType: jobType,
+                        modelId: model,
+                    });
+                }
+            }
+            this.logger.debug(`Sync completed for models ${models} from block ${fromBlock} to block ${toBlock}`);
+        });
+    }
+}
+//# sourceMappingURL=sync.js.map

package/lib/history-sync/workers/sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../src/history-sync/workers/sync.ts"],"names":[],"mappings":"AAGA,OAAO,EAAe,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAGvF,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,MAAM,CAAA;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAA;AAC5D,OAAO,EAGL,gBAAgB,EAChB,mBAAmB,GAEpB,MAAM,kBAAkB,CAAA;AAIzB,MAAM,gBAAgB,GAAgB;IACpC,UAAU,EAAE,CAAC;IACb,UAAU,EAAE,EAAE;IACd,YAAY,EAAE,IAAI;IAClB,aAAa,EAAE,EAAE;IACjB,aAAa,EAAE,CAAC;CACjB,CAAA;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAiB,EACjB,OAAqB;IAErB,OAAO;QACL,IAAI,EAAE,mBAAmB;QACzB,IAAI;QACJ,OAAO,EAAE,OAAO,IAAI,gBAAgB;KACrC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAiB,EAAE,OAAqB;IAC3E,OAAO;QACL,IAAI,EAAE,gBAAgB;QACtB,IAAI;QACJ,OAAO,EAAE,OAAO,IAAI,gBAAgB;KACrC,CAAA;AACH,CAAC;AAWD,MAAM,OAAO,UAAU;IACrB,YACmB,QAAkB,EAClB,QAA4B,EAC5B,OAAyB,EACzB,MAAyB,EACzB,oBAAsD;QAJtD,aAAQ,GAAR,QAAQ,CAAU;QAClB,aAAQ,GAAR,QAAQ,CAAoB;QAC5B,YAAO,GAAP,OAAO,CAAkB;QACzB,WAAM,GAAN,MAAM,CAAmB;QACzB,yBAAoB,GAApB,oBAAoB,CAAkC;IACtE,CAAC;IAQJ,KAAK,CAAC,OAAO,CAAC,GAAe;QAC3B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAmB,CAAA;QACvC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;QACvD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,SAAS,CAAA;QAEtD,MAAM,WAAW,GAAG,wBAAwB,CAAC;YAC3C,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,YAAY;YACvB,OAAO;SACR,CAAC,CAAC,IAAI,CAEL,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,0DAA0D,MAAM,eAAe,YAAY,aAAa,OAAO,KAAK,GAAG,EAAE,CAC1H,CAAA;YACD,OAAO,EAAE,CAAC,IAAI,CAAC,CAAA;QACjB,CAAC,CAAC,EAGF,SAAS,CAAC,KAAK,EAAE,WAA+B,EAAE,EAAE;YAClD,IAAI,CAAC,WAAW,EAAE;gBAChB,MAAM,KAAK,CAAC,2BAA2B,CAAC,CAAA;aACzC;YAED,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,WAAW,CAAA;YAC3C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAgC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC7D,sBAAsB,CAAC,KAAK,EAAE,MAAM,CAAC,CACtC,CAAA;gBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBAEjC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,wBAAwB,IAAI,CAAC,MAAM,yCAAyC,WAAW,EAAE,CAC1F,CAAA;aACF;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;gBACpC,SAAS;gBACT,YAAY,EAAE,WAAW,GAAG,CAAC;gBAC7B,OAAO;gBACP,MAAM;gBACN,OAAO;aACO,CAAC,CAAA;QACnB,CAAC,CAAC,CACH,CAAA;QAED,MAAM,aAAa,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;YACzC,IAAI,IAAI,CAAC,oBAAoB,EAAE;gBAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBAC1B,IAAI,CAAC,oBAAoB,CAAC;wBACxB,OAAO,EAAE,OAAO;wBAChB,OAAO,EAAE,KAAK;qBACf,CAAC,CAAA;iBACH;aACF;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,6BAA6B,MAAM,eAAe,SAAS,aAAa,OAAO,EAAE,CAClF,CAAA;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;CACF"}

package/lib/index-query-not-available.error.d.ts

@@ -0,0 +1,5 @@
+import type { StreamID } from '@ceramicnetwork/streamid';
+export declare class IndexQueryNotAvailableError extends Error {
+    constructor(model: StreamID | string);
+}
+//# sourceMappingURL=index-query-not-available.error.d.ts.map

package/lib/index-query-not-available.error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-query-not-available.error.d.ts","sourceRoot":"","sources":["../src/index-query-not-available.error.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAExD,qBAAa,2BAA4B,SAAQ,KAAK;gBACxC,KAAK,EAAE,QAAQ,GAAG,MAAM;CAKrC"}

package/lib/index-query-not-available.error.js

@@ -0,0 +1,6 @@
+export class IndexQueryNotAvailableError extends Error {
+    constructor(model) {
+        super(`Queries on Model ${model} are not currently available because historical data for that model is still syncing`);
+    }
+}
+//# sourceMappingURL=index-query-not-available.error.js.map

package/lib/index-query-not-available.error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-query-not-available.error.js","sourceRoot":"","sources":["../src/index-query-not-available.error.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,2BAA4B,SAAQ,KAAK;IACpD,YAAY,KAAwB;QAClC,KAAK,CACH,oBAAoB,KAAK,sFAAsF,CAChH,CAAA;IACH,CAAC;CACF"}

package/lib/index.d.ts

@@ -0,0 +1,7 @@
+export type { IndexingConfig } from './build-indexing.js';
+export { CONFIG_TABLE_NAME } from './config.js';
+export { INDEXED_MODEL_CONFIG_TABLE_NAME } from './database-index-api.js';
+export { BLOCK_CONFIRMATIONS, STATE_TABLE_NAME, SyncApi } from './history-sync/sync-api.js';
+export { LocalIndexApi } from './local-index-api.js';
+export { getDefaultCDBDatabaseConfig } from './migrations/1-create-model-table.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAA;AACzE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,2BAA2B,EAAE,MAAM,sCAAsC,CAAA"}

package/lib/index.js

@@ -0,0 +1,6 @@
+export { CONFIG_TABLE_NAME } from './config.js';
+export { INDEXED_MODEL_CONFIG_TABLE_NAME } from './database-index-api.js';
+export { BLOCK_CONFIRMATIONS, STATE_TABLE_NAME, SyncApi } from './history-sync/sync-api.js';
+export { LocalIndexApi } from './local-index-api.js';
+export { getDefaultCDBDatabaseConfig } from './migrations/1-create-model-table.js';
+//# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAA;AACzE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,2BAA2B,EAAE,MAAM,sCAAsC,CAAA"}

package/lib/insertion-order.d.ts

@@ -0,0 +1,15 @@
+import { type Knex } from 'knex';
+import { StreamID } from '@ceramicnetwork/streamid';
+import type { BaseQuery, Page, Pagination } from '@ceramicnetwork/common';
+export declare class InsertionOrder {
+    private readonly dbConnection;
+    constructor(dbConnection: Knex);
+    page(query: BaseQuery & Pagination): Promise<Page<StreamID>>;
+    private forwardQuery;
+    private backwardQuery;
+    private query;
+    applyFilters(builder: Knex.QueryBuilder, query: BaseQuery): Knex.QueryBuilder;
+    private applyCursor;
+    private applySorting;
+}
+//# sourceMappingURL=insertion-order.d.ts.map

package/lib/insertion-order.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"insertion-order.d.ts","sourceRoot":"","sources":["../src/insertion-order.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,MAAM,CAAA;AAEhC,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAsB,MAAM,wBAAwB,CAAA;AA0G7F,qBAAa,cAAc;IACb,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAAZ,YAAY,EAAE,IAAI;IAEzC,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAuDpD,YAAY;YAYZ,aAAa;IAY3B,OAAO,CAAC,KAAK;IAiBb,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,SAAS,GAAG,IAAI,CAAC,YAAY;IAuB7E,OAAO,CAAC,WAAW;IA4BnB,OAAO,CAAC,YAAY;CAoBrB"}

package/lib/insertion-order.js

@@ -0,0 +1,180 @@
+import * as uint8arrays from 'uint8arrays';
+import { StreamID } from '@ceramicnetwork/streamid';
+import { PaginationKind, parsePagination, } from './parse-pagination.js';
+import { asTableName } from './as-table-name.util.js';
+import { UnsupportedOrderingError } from './unsupported-ordering-error.js';
+import { addColumnPrefix } from './column-name.util.js';
+import { contentKey, convertQueryFilter, DATA_FIELD } from './query-filter-converter.js';
+import { parseQueryFilters } from './query-filter-parser.js';
+class Cursor {
+    static parse(cursor) {
+        return cursor
+            ? JSON.parse(uint8arrays.toString(uint8arrays.fromString(cursor, 'base64url')))
+            : undefined;
+    }
+    static stringify(input, orderByKeys = []) {
+        if (input == null) {
+            return undefined;
+        }
+        let cursor;
+        if (orderByKeys.length === 0) {
+            cursor = { type: 'timestamp', id: input.stream_id, value: input.created_at };
+        }
+        else {
+            const content = typeof input.stream_content === 'string'
+                ? JSON.parse(input.stream_content)
+                : input.stream_content;
+            cursor = { type: 'content', id: input.stream_id, value: {} };
+            for (const key of orderByKeys) {
+                if (content[key] != null) {
+                    cursor.value[key] = content[key];
+                }
+            }
+        }
+        return uint8arrays.toString(uint8arrays.fromString(JSON.stringify(cursor)), 'base64url');
+    }
+}
+const REVERSE_ORDER = {
+    ASC: 'DESC',
+    DESC: 'ASC',
+};
+function getComparisonSign(order = 'ASC', reverse = false) {
+    return order === 'ASC' ? (reverse ? '<' : '>') : reverse ? '>' : '<';
+}
+function reverseOrder(entries) {
+    return entries.map((entry) => ({ ...entry, order: REVERSE_ORDER[entry.order] }));
+}
+const INSERTION_ORDER = [{ column: 'created_at', order: 'ASC' }];
+export class InsertionOrder {
+    constructor(dbConnection) {
+        this.dbConnection = dbConnection;
+    }
+    async page(query) {
+        const orderByKeys = Object.keys(query.sorting ?? {});
+        const pagination = parsePagination(query);
+        const paginationKind = pagination.kind;
+        switch (paginationKind) {
+            case PaginationKind.FORWARD: {
+                const limit = pagination.first;
+                const response = await this.forwardQuery(query, pagination);
+                const entries = response.slice(0, limit);
+                const firstEntry = entries[0];
+                const lastEntry = entries[entries.length - 1];
+                return {
+                    edges: entries.map((row) => {
+                        return {
+                            cursor: Cursor.stringify(row, orderByKeys),
+                            node: StreamID.fromString(row.stream_id),
+                        };
+                    }),
+                    pageInfo: {
+                        hasNextPage: response.length > limit,
+                        hasPreviousPage: false,
+                        endCursor: Cursor.stringify(lastEntry, orderByKeys),
+                        startCursor: Cursor.stringify(firstEntry, orderByKeys),
+                    },
+                };
+            }
+            case PaginationKind.BACKWARD: {
+                const limit = pagination.last;
+                const response = await this.backwardQuery(query, pagination);
+                const entries = response.slice(-limit);
+                const firstEntry = entries[0];
+                const lastEntry = entries[entries.length - 1];
+                return {
+                    edges: entries.map((row) => {
+                        return {
+                            cursor: Cursor.stringify(row, orderByKeys),
+                            node: StreamID.fromString(row.stream_id),
+                        };
+                    }),
+                    pageInfo: {
+                        hasNextPage: false,
+                        hasPreviousPage: response.length > limit,
+                        endCursor: Cursor.stringify(lastEntry, orderByKeys),
+                        startCursor: Cursor.stringify(firstEntry, orderByKeys),
+                    },
+                };
+            }
+            default:
+                throw new UnsupportedOrderingError(paginationKind);
+        }
+    }
+    async forwardQuery(query, pagination) {
+        return await this.query(query, false, Cursor.parse(pagination.after)).limit(pagination.first + 1);
+    }
+    async backwardQuery(query, pagination) {
+        const response = await this.query(query, true, Cursor.parse(pagination.before)).limit(pagination.last + 1);
+        response.reverse();
+        return response;
+    }
+    query(query, isReverseOrder, cursor) {
+        let builder = this.dbConnection
+            .from(asTableName(query.model))
+            .columns(['stream_id', 'last_anchored_at', 'created_at', DATA_FIELD])
+            .select();
+        builder = this.applyFilters(builder, query);
+        const sorting = query.sorting ?? {};
+        if (cursor != null) {
+            builder = this.applyCursor(builder, cursor, isReverseOrder, sorting);
+        }
+        builder = this.applySorting(builder, isReverseOrder, sorting);
+        return builder;
+    }
+    applyFilters(builder, query) {
+        if (query.account) {
+            builder = builder.where({ controller_did: query.account });
+        }
+        if (query.queryFilters) {
+            const parsed = parseQueryFilters(query.queryFilters);
+            const converted = convertQueryFilter(parsed);
+            if (converted) {
+                builder = builder.where(converted.where);
+            }
+        }
+        else if (query.filter) {
+            for (const [key, value] of Object.entries(query.filter)) {
+                const filterObj = {};
+                filterObj[addColumnPrefix(key)] = value;
+                builder = builder.andWhere(filterObj);
+            }
+        }
+        return builder;
+    }
+    applyCursor(builder, cursor, isReverseOrder, sorting) {
+        if (cursor.type === 'timestamp') {
+            builder = builder.where((qb) => {
+                qb.where('created_at', isReverseOrder ? '<' : '>', cursor.value)
+                    .orWhere('created_at', '=', cursor.value)
+                    .andWhere('stream_id', '>', cursor.id);
+            });
+        }
+        else {
+            for (const [key, value] of Object.entries(cursor.value)) {
+                const field = contentKey(key);
+                const sign = getComparisonSign(sorting[key], isReverseOrder);
+                builder = builder.where((qb) => {
+                    qb.whereRaw(`${field} ${sign} ?`, [value])
+                        .orWhereRaw(`${field} = ?`, [value])
+                        .andWhere('stream_id', '>', cursor.id);
+                });
+            }
+        }
+        return builder;
+    }
+    applySorting(builder, isReverseOrder, sorting) {
+        const sortingEntries = Object.entries(sorting ?? {});
+        if (sortingEntries.length === 0) {
+            builder = builder.orderBy(isReverseOrder ? reverseOrder(INSERTION_ORDER) : INSERTION_ORDER);
+        }
+        else {
+            for (const [field, order] of sortingEntries) {
+                const orderBy = isReverseOrder ? REVERSE_ORDER[order] : order;
+                builder = builder.orderByRaw(`${contentKey(field)} ${orderBy}`);
+            }
+        }
+        builder = builder.orderBy('stream_id', 'asc');
+        return builder;
+    }
+}
+//# sourceMappingURL=insertion-order.js.map

package/lib/insertion-order.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"insertion-order.js","sourceRoot":"","sources":["../src/insertion-order.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAEnD,OAAO,EAGL,cAAc,EACd,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAA;AACrD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;AACxF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAA;AAyB5D,MAAe,MAAM;IAKnB,MAAM,CAAC,KAAK,CACV,MAA0B;QAE1B,OAAO,MAAM;YACX,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;YAC/E,CAAC,CAAC,SAAS,CAAA;IACf,CAAC;IAMD,MAAM,CAAC,SAAS,CACd,KAA8B,EAC9B,cAA6B,EAAE;QAE/B,IAAI,KAAK,IAAI,IAAI,EAAE;YACjB,OAAO,SAAS,CAAA;SACjB;QAED,IAAI,MAAkB,CAAA;QACtB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE;YAE5B,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE,CAAA;SAC7E;aAAM;YAEL,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;gBACtC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC;gBAClC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAA;YAC1B,MAAM,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;YAC5D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE;gBAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE;oBACxB,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;iBACjC;aACF;SACF;QAED,OAAO,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;IAC1F,CAAC;CACF;AAED,MAAM,aAAa,GAAiC;IAClD,GAAG,EAAE,MAAM;IACX,IAAI,EAAE,KAAK;CACZ,CAAA;AAID,SAAS,iBAAiB,CAAC,QAAmB,KAAK,EAAE,OAAO,GAAG,KAAK;IAClE,OAAO,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;AACtE,CAAC;AAKD,SAAS,YAAY,CAAiC,OAAiB;IACrE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;AAClF,CAAC;AAED,MAAM,eAAe,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,KAAc,EAAE,CAAC,CAAA;AAKzE,MAAM,OAAO,cAAc;IACzB,YAA6B,YAAkB;QAAlB,iBAAY,GAAZ,YAAY,CAAM;IAAG,CAAC;IAEnD,KAAK,CAAC,IAAI,CAAC,KAA6B;QACtC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACpD,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAA;QACtC,QAAQ,cAAc,EAAE;YACtB,KAAK,cAAc,CAAC,OAAO,CAAC,CAAC;gBAC3B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAA;gBAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;gBAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;gBACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAC7C,OAAO;oBACL,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBACzB,OAAO;4BACL,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAE;4BAC3C,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;yBACzC,CAAA;oBACH,CAAC,CAAC;oBACF,QAAQ,EAAE;wBACR,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,KAAK;wBACpC,eAAe,EAAE,KAAK;wBACtB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,WAAW,CAAC;wBACnD,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE,WAAW,CAAC;qBACvD;iBACF,CAAA;aACF;YACD,KAAK,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAC5B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAA;gBAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;gBAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAA;gBACtC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAC7C,OAAO;oBACL,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBACzB,OAAO;4BACL,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAE;4BAC3C,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;yBACzC,CAAA;oBACH,CAAC,CAAC;oBACF,QAAQ,EAAE;wBACR,WAAW,EAAE,KAAK;wBAClB,eAAe,EAAE,QAAQ,CAAC,MAAM,GAAG,KAAK;wBACxC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,WAAW,CAAC;wBACnD,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE,WAAW,CAAC;qBACvD;iBACF,CAAA;aACF;YACD;gBACE,MAAM,IAAI,wBAAwB,CAAC,cAAc,CAAC,CAAA;SACrD;IACH,CAAC;IAKO,KAAK,CAAC,YAAY,CACxB,KAAgB,EAChB,UAAkC;QAElC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CACzE,UAAU,CAAC,KAAK,GAAG,CAAC,CACrB,CAAA;IACH,CAAC;IAKO,KAAK,CAAC,aAAa,CACzB,KAAgB,EAChB,UAAmC;QAEnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CACnF,UAAU,CAAC,IAAI,GAAG,CAAC,CACpB,CAAA;QAED,QAAQ,CAAC,OAAO,EAAE,CAAA;QAClB,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,KAAgB,EAAE,cAAuB,EAAE,MAAmB;QAC1E,IAAI,OAAO,GAAiB,IAAI,CAAC,YAAY;aAC1C,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;aAC9B,OAAO,CAAC,CAAC,WAAW,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;aACpE,MAAM,EAAE,CAAA;QAEX,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAA;QAEnC,IAAI,MAAM,IAAI,IAAI,EAAE;YAClB,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;SACrE;QAED,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAA;QAC7D,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,YAAY,CAAC,OAA0B,EAAE,KAAgB;QACvD,IAAI,KAAK,CAAC,OAAO,EAAE;YACjB,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;SAC3D;QAED,IAAI,KAAK,CAAC,YAAY,EAAE;YACtB,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YACpD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;YAC5C,IAAI,SAAS,EAAE;gBACb,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;aACzC;SACF;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE;gBACvD,MAAM,SAAS,GAA2B,EAAE,CAAA;gBAC5C,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAA;gBACvC,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;aACtC;SACF;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,WAAW,CACjB,OAAqB,EACrB,MAAkB,EAClB,cAAuB,EACvB,OAAgB;QAEhB,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE;YAE/B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;gBAC7B,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC;qBAC7D,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC;qBACxC,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;YAC1C,CAAC,CAAC,CAAA;SACH;aAAM;YAEL,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBACvD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAA;gBAC7B,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAA;gBAC5D,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;oBAC7B,EAAE,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;yBACvC,UAAU,CAAC,GAAG,KAAK,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;yBACnC,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;gBAC1C,CAAC,CAAC,CAAA;aACH;SACF;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,YAAY,CAClB,OAAqB,EACrB,cAAuB,EACvB,OAAgB;QAEhB,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACpD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;YAE/B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAA;SAC5F;aAAM;YAEL,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,cAAc,EAAE;gBAC3C,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;gBAC7D,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC,CAAA;aAChE;SACF;QAED,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QAC7C,OAAO,OAAO,CAAA;IAChB,CAAC;CACF"}

package/lib/local-index-api.d.ts

@@ -0,0 +1,30 @@
+import type { BaseQuery, IndexApi, ModelData, Page, PaginationQuery, StreamState, DiagnosticsLogger, LoadOpts, Stream } from '@ceramicnetwork/common';
+import type { IndexModelArgs } from './database-index-api.js';
+import { IndexStreamArgs } from './database-index-api.js';
+import { type CommitID, StreamID } from '@ceramicnetwork/streamid';
+import { IndexingConfig } from './build-indexing.js';
+import { Networks } from '@ceramicnetwork/common';
+import { ISyncQueryApi } from './history-sync/interfaces.js';
+export declare type CeramicCoreApi = {
+    loadStream<T extends Stream>(streamId: StreamID | CommitID | string, opts?: LoadOpts): Promise<T>;
+    loadStreamState(streamId: StreamID): Promise<StreamState | undefined>;
+};
+export declare class LocalIndexApi implements IndexApi {
+    private readonly core;
+    private readonly logger;
+    private readonly databaseIndexApi;
+    readonly enabled: boolean;
+    constructor(indexingConfig: IndexingConfig | undefined, core: CeramicCoreApi, logger: DiagnosticsLogger, networkName: Networks);
+    setSyncQueryApi(api: ISyncQueryApi): void;
+    shouldIndexStream(args: StreamID): boolean;
+    indexStream(args: IndexStreamArgs): Promise<void>;
+    count(query: BaseQuery): Promise<number>;
+    query(query: PaginationQuery): Promise<Page<StreamState | null>>;
+    indexedModels(): Array<ModelData>;
+    convertModelDataToIndexModelsArgs(modelsNoLongerIndexed: Array<ModelData>, modelData: ModelData): Promise<IndexModelArgs>;
+    indexModels(models: Array<ModelData>): Promise<void>;
+    stopIndexingModels(models: Array<ModelData>): Promise<void>;
+    init(): Promise<void>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=local-index-api.d.ts.map

package/lib/local-index-api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-index-api.d.ts","sourceRoot":"","sources":["../src/local-index-api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,QAAQ,EACR,MAAM,EACP,MAAM,wBAAwB,CAAA;AAC/B,OAAO,KAAK,EAAoB,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,KAAK,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAA;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAEpD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAE5D,oBAAY,cAAc,GAAG;IAC3B,UAAU,CAAC,CAAC,SAAS,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IACjG,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAA;CACtE,CAAA;AAgCD,qBAAa,aAAc,YAAW,QAAQ;IAM1C,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA8B;IAC/D,SAAgB,OAAO,EAAE,OAAO,CAAA;gBAG9B,cAAc,EAAE,cAAc,GAAG,SAAS,EACzB,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,iBAAiB,EAC1C,WAAW,EAAE,QAAQ;IAMvB,eAAe,CAAC,GAAG,EAAE,aAAa;IAMlC,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO;IAcpC,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAQjD,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IAUxC,KAAK,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAwCtE,aAAa,IAAI,KAAK,CAAC,SAAS,CAAC;IAI3B,iCAAiC,CACrC,qBAAqB,EAAE,KAAK,CAAC,SAAS,CAAC,EACvC,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,cAAc,CAAC;IAiBpB,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpD,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3D,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAYrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}

package/lib/local-index-api.js

@@ -0,0 +1,118 @@
+import { makeIndexApi } from './make-index-api.js';
+import { Model } from '@ceramicnetwork/stream-model';
+async function _getIndexModelArgs(req, core) {
+    const modelStreamId = req.streamID;
+    if (modelStreamId.type != Model.STREAM_TYPE_ID && !modelStreamId.equals(Model.MODEL)) {
+        throw new Error(`Cannot index ${modelStreamId.toString()}, it is not a Model StreamID`);
+    }
+    const opts = {
+        model: modelStreamId,
+    };
+    if (modelStreamId.type == Model.STREAM_TYPE_ID) {
+        const modelState = await core.loadStream(modelStreamId, {});
+        const content = modelState.state.next?.content ?? modelState.state.content;
+        Model.assertVersionValid(content, 'major');
+        if (content.relations) {
+            opts.relations = content.relations;
+        }
+        opts.indices = req.indices;
+    }
+    return opts;
+}
+export class LocalIndexApi {
+    constructor(indexingConfig, core, logger, networkName) {
+        this.core = core;
+        this.logger = logger;
+        this.databaseIndexApi = makeIndexApi(indexingConfig, networkName, logger);
+        this.enabled = indexingConfig != null && !indexingConfig.disableComposedb;
+    }
+    setSyncQueryApi(api) {
+        if (this.databaseIndexApi) {
+            this.databaseIndexApi.setSyncQueryApi(api);
+        }
+    }
+    shouldIndexStream(args) {
+        if (!this.databaseIndexApi) {
+            return false;
+        }
+        return this.databaseIndexApi.getIndexedModels().some(function (idx) {
+            return idx.streamID.equals(args);
+        });
+    }
+    async indexStream(args) {
+        if (!this.shouldIndexStream(args.model)) {
+            return;
+        }
+        await this.databaseIndexApi.indexStream(args);
+    }
+    async count(query) {
+        return this.databaseIndexApi.count(query);
+    }
+    async query(query) {
+        if (!this.databaseIndexApi) {
+            this.logger.warn(`Indexing is not configured. Unable to serve query ${JSON.stringify(query)}`);
+            return {
+                edges: [],
+                pageInfo: {
+                    hasNextPage: false,
+                    hasPreviousPage: false,
+                },
+            };
+        }
+        const page = await this.databaseIndexApi.page(query);
+        const edges = await Promise.all(page.edges.map(async (edge) => {
+            const node = (await this.core.loadStreamState(edge.node)) ?? null;
+            if (!node) {
+                this.logger.warn(`
+            Did not find stream state for streamid ${edge.node} in our state store when serving an indexed query.
+            This may indicate a problem with data persistence of your state store, which can result in data loss.
+            Please check that your state store is properly configured with strong persistence guarantees.
+            This query may have incomplete results. Affected query: ${JSON.stringify(query)}
+            `);
+            }
+            return {
+                cursor: edge.cursor,
+                node: node,
+            };
+        }));
+        return {
+            edges: edges,
+            pageInfo: page.pageInfo,
+        };
+    }
+    indexedModels() {
+        return this.databaseIndexApi?.getIndexedModels() ?? [];
+    }
+    async convertModelDataToIndexModelsArgs(modelsNoLongerIndexed, modelData) {
+        const modelStreamId = modelData.streamID;
+        this.logger.imp(`Starting indexing for Model ${modelStreamId.toString()}`);
+        const modelNoLongerIndexed = modelsNoLongerIndexed.some(function (oldIdx) {
+            return oldIdx.streamID.equals(modelStreamId);
+        });
+        if (modelNoLongerIndexed) {
+            throw new Error(`Cannot re-index model ${modelStreamId.toString()}, data may not be up-to-date`);
+        }
+        return await _getIndexModelArgs(modelData, this.core);
+    }
+    async indexModels(models) {
+        const modelsNoLongerIndexed = (await this.databaseIndexApi?.getModelsNoLongerIndexed()) ?? [];
+        const indexModelsArgs = await Promise.all(models.map(async (idx) => await this.convertModelDataToIndexModelsArgs(modelsNoLongerIndexed, idx)));
+        await this.databaseIndexApi?.indexModels(indexModelsArgs);
+    }
+    async stopIndexingModels(models) {
+        this.logger.imp(`Stopping indexing for Models: ${models.map(String).join(',')}`);
+        await this.databaseIndexApi?.stopIndexingModels(models);
+    }
+    async init() {
+        if (!this.databaseIndexApi) {
+            return;
+        }
+        await this.databaseIndexApi.init();
+        const modelsToIndex = this.databaseIndexApi.getIndexedModels();
+        await this.indexModels(modelsToIndex);
+    }
+    async close() {
+        await this.databaseIndexApi?.close();
+    }
+}
+//# sourceMappingURL=local-index-api.js.map

package/lib/local-index-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-index-api.js","sourceRoot":"","sources":["../src/local-index-api.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAElD,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAA;AAYpD,KAAK,UAAU,kBAAkB,CAAC,GAAc,EAAE,IAAoB;IACpE,MAAM,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAA;IAClC,IAAI,aAAa,CAAC,IAAI,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;QACpF,MAAM,IAAI,KAAK,CAAC,gBAAgB,aAAa,CAAC,QAAQ,EAAE,8BAA8B,CAAC,CAAA;KACxF;IAED,MAAM,IAAI,GAAmB;QAC3B,KAAK,EAAE,aAAa;KACrB,CAAA;IAED,IAAI,aAAa,CAAC,IAAI,IAAI,KAAK,CAAC,cAAc,EAAE;QAC9C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAA;QAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,CAAA;QAC1E,KAAK,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAC1C,IAAI,OAAO,CAAC,SAAS,EAAE;YACrB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;SACnC;QACD,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAA;KAC3B;IAED,OAAO,IAAI,CAAA;AACb,CAAC;AAKD,MAAM,OAAO,aAAa;IAIxB,YACE,cAA0C,EACzB,IAAoB,EACpB,MAAyB,EAC1C,WAAqB;QAFJ,SAAI,GAAJ,IAAI,CAAgB;QACpB,WAAM,GAAN,MAAM,CAAmB;QAG1C,IAAI,CAAC,gBAAgB,GAAG,YAAY,CAAC,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;QACzE,IAAI,CAAC,OAAO,GAAG,cAAc,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAA;IAC3E,CAAC;IAED,eAAe,CAAC,GAAkB;QAChC,IAAI,IAAI,CAAC,gBAAgB,EAAE;YACzB,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;SAC3C;IACH,CAAC;IAED,iBAAiB,CAAC,IAAc;QAC9B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAC1B,OAAO,KAAK,CAAA;SACb;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,GAAG;YAChE,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAClC,CAAC,CAAC,CAAA;IACJ,CAAC;IAMD,KAAK,CAAC,WAAW,CAAC,IAAqB;QAErC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;YACvC,OAAM;SACP;QACD,MAAM,IAAI,CAAC,gBAAiB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;IAChD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAgB;QAC1B,OAAO,IAAI,CAAC,gBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAC5C,CAAC;IAQD,KAAK,CAAC,KAAK,CAAC,KAAsB;QAChC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAC9F,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE;oBACR,WAAW,EAAE,KAAK;oBAClB,eAAe,EAAE,KAAK;iBACvB;aACF,CAAA;SACF;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACpD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAE7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAA;YACjE,IAAI,CAAC,IAAI,EAAE;gBACT,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;qDAEb,IAAI,CAAC,IACP;;;sEAG0D,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAC9E,CAAC,CAAA;aACL;YAED,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,IAAI,EAAE,IAAI;aACX,CAAA;QACH,CAAC,CAAC,CACH,CAAA;QACD,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAA;IACH,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAA;IACxD,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,qBAAuC,EACvC,SAAoB;QAEpB,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAA;QACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAE1E,MAAM,oBAAoB,GAAG,qBAAqB,CAAC,IAAI,CAAC,UAAU,MAAM;YACtE,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QAEF,IAAI,oBAAoB,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yBAAyB,aAAa,CAAC,QAAQ,EAAE,8BAA8B,CAChF,CAAA;SACF;QAED,OAAO,MAAM,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IACvD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAwB;QACxC,MAAM,qBAAqB,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,wBAAwB,EAAE,CAAC,IAAI,EAAE,CAAA;QAE7F,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,MAAM,CAAC,GAAG,CACR,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,iCAAiC,CAAC,qBAAqB,EAAE,GAAG,CAAC,CACxF,CACF,CAAA;QAED,MAAM,IAAI,CAAC,gBAAgB,EAAE,WAAW,CAAC,eAAe,CAAC,CAAA;IAC3D,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAwB;QAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iCAAiC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAChF,MAAM,IAAI,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAA;IACzD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAC1B,OAAM;SACP;QACD,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAA;QAIlC,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAA;QAC9D,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,CAAA;IACvC,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAA;IACtC,CAAC;CACF"}

package/lib/make-index-api.d.ts

@@ -0,0 +1,9 @@
+import type { DiagnosticsLogger } from '@ceramicnetwork/common';
+import { Networks } from '@ceramicnetwork/common';
+import { IndexingConfig } from './build-indexing.js';
+import { DatabaseIndexApi } from './database-index-api.js';
+export declare class UnsupportedIndexingDatabaseError extends Error {
+    constructor();
+}
+export declare function makeIndexApi(indexingConfig: IndexingConfig | undefined, network: Networks, logger: DiagnosticsLogger): DatabaseIndexApi | undefined;
+//# sourceMappingURL=make-index-api.d.ts.map

package/lib/make-index-api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-index-api.d.ts","sourceRoot":"","sources":["../src/make-index-api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAiB,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACnE,OAAO,EAAE,gBAAgB,EAAkB,MAAM,yBAAyB,CAAA;AAE1E,qBAAa,gCAAiC,SAAQ,KAAK;;CAM1D;AAKD,wBAAgB,YAAY,CAC1B,cAAc,EAAE,cAAc,GAAG,SAAS,EAC1C,OAAO,EAAE,QAAQ,EACjB,MAAM,EAAE,iBAAiB,GACxB,gBAAgB,GAAG,SAAS,CA2B9B"}

package/lib/make-index-api.js

@@ -0,0 +1,27 @@
+import { Networks } from '@ceramicnetwork/common';
+import { buildIndexing } from './build-indexing.js';
+import { SqliteIndexApi } from './database-index-api.js';
+export class UnsupportedIndexingDatabaseError extends Error {
+    constructor() {
+        super('SQLite is not supported for the ComposeDB indexing database in production use. Please setup a Postgres instance and update the config file to use ComposeDB, or disable ComposeDB to start up without an indexing database.');
+    }
+}
+export function makeIndexApi(indexingConfig, network, logger) {
+    if (!indexingConfig) {
+        logger.warn(`ComposeDB Indexing is not configured. Please add the indexing settings to your config file`);
+        return undefined;
+    }
+    if (indexingConfig.disableComposedb === true) {
+        logger.warn('ComposeDB indexing is actively disabled. Change the corresponding CLI flag, config option or ENV variable to enable it.');
+        return undefined;
+    }
+    const indexApi = buildIndexing(indexingConfig, logger, network);
+    if (indexingConfig.enableHistoricalSync && indexApi instanceof SqliteIndexApi) {
+        throw new UnsupportedIndexingDatabaseError();
+    }
+    if (network === Networks.MAINNET && indexApi instanceof SqliteIndexApi) {
+        throw new UnsupportedIndexingDatabaseError();
+    }
+    return indexApi;
+}
+//# sourceMappingURL=make-index-api.js.map

package/lib/make-index-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-index-api.js","sourceRoot":"","sources":["../src/make-index-api.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAkB,MAAM,qBAAqB,CAAA;AACnE,OAAO,EAAoB,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAE1E,MAAM,OAAO,gCAAiC,SAAQ,KAAK;IACzD;QACE,KAAK,CACH,6NAA6N,CAC9N,CAAA;IACH,CAAC;CACF;AAKD,MAAM,UAAU,YAAY,CAC1B,cAA0C,EAC1C,OAAiB,EACjB,MAAyB;IAEzB,IAAI,CAAC,cAAc,EAAE;QACnB,MAAM,CAAC,IAAI,CACT,4FAA4F,CAC7F,CAAA;QACD,OAAO,SAAS,CAAA;KACjB;IAED,IAAI,cAAc,CAAC,gBAAgB,KAAK,IAAI,EAAE;QAC5C,MAAM,CAAC,IAAI,CACT,yHAAyH,CAC1H,CAAA;QACD,OAAO,SAAS,CAAA;KACjB;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IAE/D,IAAI,cAAc,CAAC,oBAAoB,IAAI,QAAQ,YAAY,cAAc,EAAE;QAC7E,MAAM,IAAI,gCAAgC,EAAE,CAAA;KAC7C;IAGD,IAAI,OAAO,KAAK,QAAQ,CAAC,OAAO,IAAI,QAAQ,YAAY,cAAc,EAAE;QACtE,MAAM,IAAI,gCAAgC,EAAE,CAAA;KAC7C;IAED,OAAO,QAAQ,CAAA;AACjB,CAAC"}

package/lib/migrations/1-create-model-table.d.ts

@@ -0,0 +1,34 @@
+import type { Knex } from 'knex';
+import { FieldsIndex, Networks } from '@ceramicnetwork/common';
+export declare enum DatabaseType {
+    POSTGRES = "postgres",
+    SQLITE = "sqlite"
+}
+export declare enum ColumnType {
+    STRING = 0
+}
+export declare type ColumnInfo = {
+    name: string;
+    type: ColumnType;
+};
+export declare type TableIndex = {
+    keys: Array<string>;
+    name: string;
+    indexType: Knex.storageEngineIndexType;
+};
+export declare type TableIndices = {
+    indexName: string;
+    indices: Array<TableIndex>;
+};
+export declare function indexNameFromTableName(tableName: string): string;
+export declare function defaultIndices(tableName: string): TableIndices;
+export declare function getDefaultCDBDatabaseConfig(networkName: string): {
+    [key: string]: any;
+};
+export declare function createPostgresModelTable(dataSource: Knex, tableName: string, extraColumns: Array<ColumnInfo>): Promise<void>;
+export declare function createSqliteModelTable(dataSource: Knex, tableName: string, extraColumns: Array<ColumnInfo>): Promise<void>;
+export declare function createPostgresIndices(dataSource: Knex, tableName: string, indices: Array<FieldsIndex>): Promise<void>;
+export declare function createSqliteIndices(dataSource: Knex, tableName: string, indices: Array<FieldsIndex>): Promise<void>;
+export declare function createConfigTable(dataSource: Knex, tableName: string, network: Networks, jsonb_support: boolean): Promise<void>;
+export declare function migrateConfigTable(dataSource: Knex, tableName: string, jsonb_support: boolean): Promise<void>;
+//# sourceMappingURL=1-create-model-table.d.ts.map

package/lib/migrations/1-create-model-table.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"1-create-model-table.d.ts","sourceRoot":"","sources":["../../src/migrations/1-create-model-table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAChC,OAAO,EAAwB,WAAW,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAKpF,oBAAY,YAAY;IACtB,QAAQ,aAAa;IACrB,MAAM,WAAW;CAClB;AAQD,oBAAY,UAAU;IACpB,MAAM,IAAA;CACP;AAKD,oBAAY,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,UAAU,CAAA;CACjB,CAAA;AAED,oBAAY,UAAU,GAAG;IACvB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IACnB,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,IAAI,CAAC,sBAAsB,CAAA;CACvC,CAAA;AAED,oBAAY,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;CAC3B,CAAA;AAMD,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEhE;AAKD,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,CA0C9D;AAWD,wBAAgB,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAE,CA0CvF;AAoBD,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,IAAI,EAChB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,GAC9B,OAAO,CAAC,IAAI,CAAC,CAwBf;AAED,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,IAAI,EAChB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,GAC9B,OAAO,CAAC,IAAI,CAAC,CAqBf;AAOD,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,IAAI,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,mBAAmB,CACvC,UAAU,EAAE,IAAI,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,GAC1B,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,IAAI,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,QAAQ,EACjB,aAAa,EAAE,OAAO,iBAgDvB;AAED,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,IAAI,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,OAAO,iBAoBvB"}