@delali/narsil 0.3.1-next.72 → 0.3.1-next.74

package/dist/embeddings/openai.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from '../chunk-RV77FR2F.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-I64LXKO3.mjs';
 
 // src/embeddings/constants.ts
 var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);

package/dist/{engine-Dm4446j1.d.ts → engine-B8eRPO7m.d.ts}

@@ -1,171 +1,6 @@
 import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
-import { B as BatchResult, I as IndexInfo, f as IndexStats, P as PartitionStatsResult, L as ListParams, g as ListResult, Q as QueryParams, e as QueryResult, h as PreflightResult, j as SuggestParams, k as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult } from './memory-CKHTU1CL.js';
-import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-BBOCdhND.js';
-
-/**
- * This map lists every event that the engine emits, with the payload of each
- * one.
- *
- * {@link Narsil.on} registers a listener under a key from this map, so that the
- * engine can pass the matching payload to that listener. Most of these events
- * report work that the engine does in the background, away from the call that
- * started it, so a failure that such an event reports never rejects a promise
- * that you are holding.
- *
- * @public
- */
-type NarsilEventMap = {
-    /**
-     * The engine emits this event after a call changes what a read returns from
-     * an index. The calls are the inserts, updates, and removals, singly or in a
-     * batch, together with `clear`, `restore`, `rebalance`, `rebuildAnalysis`,
-     * `compactVectors`, `optimizeVectors`, `createIndex`, and `dropIndex`. The
-     * engine emits the event after a call that throws as well, because a call can
-     * change the index before it throws. It also emits the event after it reloads
-     * an index, once a shared invalidation adapter reports that another instance
-     * saved that index. The engine emits one event per batch, however many
-     * documents the batch holds.
-     *
-     * The engine emits the event once a query can see the change, which means
-     * after every worker copy applies the change and after any rebalance in
-     * progress replays it. A query that a listener sends from its handler
-     * therefore returns results that include the change. The engine waits for the
-     * change to become visible only while a listener is registered for the event.
-     */
-    write: {
-        /** This is the name of the index that changed. */
-        indexName: string;
-    };
-    /**
-     * A worker thread exited unexpectedly. The pool removes the thread and fails
-     * the requests that the thread was serving with `WORKER_CRASHED`. The engine
-     * then sends each of those queries again on the main thread, so the caller
-     * receives a result with no error. The remaining workers keep serving
-     * queries, because each of them holds a full worker copy of every promoted
-     * index.
-     *
-     * After a delay, the engine spawns a replacement and loads every copy onto
-     * it before it sends queries to the replacement. Once no worker is left, the
-     * main thread serves every query, because it holds every document. The first
-     * request after the delay then starts a new pool. The delay starts at one
-     * second and doubles up to a minute while replacements keep failing.
-     */
-    workerCrash: {
-        /** This is the id of the worker that exited. */
-        workerId: number;
-        /** These are the indexes that the worker held. */
-        indexNames: string[];
-        /** This is the error that ended the worker. */
-        error: Error;
-    };
-    /** An index gained worker copies, either because it reached the copy
-     * threshold or because a request loaded its copies again after an idle
-     * spell. */
-    workerPromote: {
-        /** This is the number of workers in the pool. */
-        workerCount: number;
-        /** This is the reason that the index gained its copies, such as the copy
-         * threshold that it reached. */
-        reason: string;
-    };
-    /** An index could not gain worker copies, so the main thread keeps serving
-     * its queries. */
-    workerPromoteFailure: {
-        /** This is the reason that the engine tried to load the copies. */
-        reason: string;
-        /** This is the error that stopped the load. */
-        error: Error;
-        /** This is true when the engine will try again at a later threshold. */
-        retryable: boolean;
-    };
-    /** An index finished spreading its documents across a new number of
-     * partitions. */
-    partitionRebalance: {
-        /** This is the name of the index that the engine rebalanced. */
-        indexName: string;
-        /** This is the number of partitions before the rebalance. */
-        oldCount: number;
-        /** This is the number of partitions after the rebalance. */
-        newCount: number;
-    };
-    /**
-     * An index passed its watermark, so its partitions are close to their
-     * capacity. On a cluster node, the engine reports the document count and the
-     * capacity of the one partition that `partitionId` identifies, which this
-     * node leads.
-     */
-    partitionWatermark: {
-        /** This is the name of the index that crossed the mark. */
-        indexName: string;
-        /** This is the number of documents in the index, or in the partition on a
-         * cluster node. */
-        documentCount: number;
-        /** This is the number of documents that the current partitions of the
-         * index can hold, or that the partition can hold on a cluster node. */
-        capacity: number;
-        /** This is the number of partitions in the index. */
-        partitionCount: number;
-        /** On a cluster node, this is the partition that crossed the mark, while
-         * outside a cluster the field is absent. */
-        partitionId?: number;
-    };
-    /**
-     * The process uses nine tenths of its heap, as measured during a write or a
-     * load, so the next large index can end the process with an out-of-memory
-     * error.
-     *
-     * Where `--max-old-space-size` or `--max-old-space-size-percentage` sets a
-     * limit, on the command line or in `NODE_OPTIONS`, the engine measures the
-     * used bytes against that limit, which `heapLimit` then reports. Where
-     * neither flag sets a limit, the engine measures the share of the heap that
-     * V8 reports as no longer available. V8 reports a limit about 192 MB above
-     * the true ceiling, so a process on a default heap below about 2 GB can end before
-     * this event fires.
-     *
-     * The engine emits the event once per crossing, then arms it again once the
-     * free share of the heap recovers to two tenths. Raise the limit with
-     * `--max-old-space-size-percentage` or `--max-old-space-size`, or close an
-     * idle index; see {@link ProcessMemoryReport.heapLimit}.
-     */
-    heapPressure: {
-        /** This is the index that a write or a load was touching when the heap
-         * reached that point. */
-        indexName: string;
-        /** This is the heap that the process uses, in bytes. */
-        heapUsed: number;
-        /** This is the size in bytes that the heap can grow to. */
-        heapLimit: number;
-        /** This is the estimate of the bytes that the index holds. */
-        estimatedMemoryBytes: number;
-    };
-    /** The engine started, completed, or failed a rebuild that brings the terms
-     * of an index up to the current analysis of its language module. */
-    analysisRebuild: {
-        /** This is the name of the index under rebuild. */
-        indexName: string;
-        /** This is the stage that the rebuild has reached. */
-        status: 'started' | 'completed' | 'failed';
-        /** This is the number of partitions that the rebuild has covered so far. */
-        partitionsRebuilt: number;
-        /** This is the number of partitions that the rebuild covers in total. */
-        partitionCount: number;
-        /** This is the error that stopped the rebuild, which only a `failed`
-         * status includes. */
-        error?: Error;
-    };
-    /** A write to the write-ahead log or a checkpoint failed, so the writes since
-     * the last checkpoint are at risk. */
-    durabilityError: {
-        /** This is the error that the durability layer threw. */
-        error: Error;
-    };
-    /** The invalidation channel failed, so this instance can serve partitions
-     * that another instance has since changed. */
-    invalidationError: {
-        /** This is the error that the adapter threw. */
-        error: Error;
-    };
-};
+import { B as BatchResult, I as IndexInfo, g as IndexStats, P as PartitionStatsResult, L as ListParams, h as ListResult, Q as QueryParams, e as QueryResult, j as PreflightResult, k as SuggestParams, l as SuggestResult, D as DocumentImportResult, M as MemoryStats, V as VectorMaintenanceResult, m as NarsilEventMap } from './memory-DJEwWoCi.js';
+import { A as AnyDocument, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-DgyDijhf.js';
 
 /**
  * Opens and closes durable indexes without deleting them.
@@ -293,6 +128,14 @@ interface DocumentWriteOperations {
     waitForWrites(indexName: string): Promise<void>;
 }
 
+/**
+ * The bytes that {@link Narsil.restore} and {@link Narsil.importDocumentExport}
+ * accept, as one array, a stream of chunks, or an async iterable of chunks,
+ * such as a Node.js file stream.
+ *
+ * @public
+ */
+type ByteSource = Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>;
 /**
  * The search engine, and everything you do with it.
  *
@@ -472,28 +315,94 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
      */
     rebuildAnalysis(indexName: string): Promise<void>;
     /**
-     * Serialises a whole index into the portable `.nrsl` format, which any
-     * Narsil implementation reads.
+     * Returns a stream of the index's snapshot archive. The archive is a pax
+     * file containing the index's configuration and the files of one
+     * checkpoint, and any implementation of the Narsil format can restore it.
+     *
+     * With write-ahead log durability, the engine takes a checkpoint before it
+     * starts the stream, so the archive contains every write that the engine
+     * acknowledged before the call. The engine keeps that checkpoint's files on
+     * disk until the stream ends or you cancel it. Without write-ahead log
+     * durability, the engine builds the archive's files from the index in
+     * memory.
+     *
+     * @param indexName - The index to stream.
+     * @returns A stream of the archive's bytes, which you can write to a file or
+     * send over a network.
+     */
+    snapshot(indexName: string): Promise<ReadableStream<Uint8Array>>;
+    /**
+     * Creates an index from a snapshot under the name that you pass, replacing
+     * any index of that name.
+     *
+     * The engine accepts the archive that {@link Narsil.snapshot} streams. It
+     * also accepts the single `.nrsl` file that `snapshot` returned in release
+     * 0.3, and it inserts that file's documents and vectors as new writes. The
+     * engine reads an archive one file at a time. Before it replaces an existing
+     * index, it checks each file against the archive's manifest and decodes
+     * every document, so a failed check leaves the existing index unchanged.
+     *
+     * @param indexName - The name that the restored index takes.
+     * @param data - The snapshot's bytes, as one array, a stream, or an async
+     * iterable of chunks.
+     * @throws A `NarsilError` with `PERSISTENCE_LOAD_FAILED` when the archive is
+     * malformed or a file's length differs from the manifest. The engine raises
+     * `PERSISTENCE_CRC_MISMATCH` when a file's checksum differs from its
+     * contents, and `ENVELOPE_VERSION_MISMATCH` when a file has a format version
+     * other than the ones that this release implements. It raises
+     * `CONFIG_INVALID` when the archive's configuration is invalid, and
+     * `DOC_VALIDATION_FAILED` when the data is empty or the release 0.3 file is
+     * invalid.
+     */
+    restore(indexName: string, data: ByteSource): Promise<void>;
+    /**
+     * Returns a stream of the index's document export. The export is JSON Lines
+     * text whose format stays the same in every release, so you can use it to
+     * move an index across releases or between implementations.
+     *
+     * The first line is a header containing the format name, the format
+     * version, and the index configuration. Each later line contains one
+     * document, with its ID under `id` and the value of each vector field as an
+     * array of numbers. With write-ahead log durability, the engine exports every
+     * document from one checkpoint; otherwise, it exports them from the index in
+     * memory at one moment.
+     *
+     * JSON has no type for a date, for raw bytes, or for an infinite number or
+     * `NaN`, so the engine writes each such value as a tag. Each tag is an
+     * object with one key that begins with `$`, such as
+     * `{"$date":[1700000000,123000000]}`. The engine also wraps a stored object
+     * whose only key begins with `$` in an `$object` tag, because the import
+     * decodes every other object of that shape as a tag.
+     *
+     * @param indexName - The index to export.
+     * @returns A stream of the export's UTF-8 bytes.
+     */
+    exportDocuments(indexName: string): Promise<ReadableStream<Uint8Array>>;
+    /**
+     * Imports a document export into the named index. Where the name is free,
+     * the engine creates the index from the export's configuration.
+     *
+     * The engine writes the documents in batches and replaces any document
+     * whose ID the index already contains. Because it replaces existing
+     * documents, you can run an interrupted import again to make the index equal
+     * to the export. When a line is invalid or the engine rejects its document,
+     * the engine records the refusal and continues with the next line.
+     *
+     * The engine converts each tag that {@link Narsil.exportDocuments} writes
+     * into the value that the tag stands for. When an object has one key that
+     * begins with `$` and that key names no tag, or when a tag holds an invalid
+     * value, the engine refuses the line.
      *
-     * @param indexName - The index to serialise.
-     * @returns The file's bytes, ready to store or send.
-     */
-    snapshot(indexName: string): Promise<Uint8Array>;
-    /**
-     * Loads an index from snapshot bytes, replacing whatever the name held.
-     *
-     * The engine reads the `.nrsl` envelope that {@link Narsil.snapshot} writes.
-     *
-     * @param indexName - The name the restored index takes.
-     * @param data - Bytes that {@link Narsil.snapshot} produced.
-     * @throws A `NarsilError` with `ENVELOPE_INVALID_MAGIC` when the bytes are
-     * shorter than an envelope header or open with anything other than the
-     * `NRSL` magic bytes, `ENVELOPE_VERSION_MISMATCH` when the envelope carries a
-     * format version other than the index snapshot format,
-     * `PERSISTENCE_CRC_MISMATCH` when the payload fails its checksum, and
-     * `DOC_VALIDATION_FAILED` when the envelope holds no valid index snapshot.
+     * @param indexName - The index that receives the documents.
+     * @param data - The export's bytes, as one array, a stream, or an async
+     * iterable of chunks.
+     * @returns The number of imported documents, the number of refused lines,
+     * and the 1,000 refusals with the lowest line numbers.
+     * @throws A `NarsilError` with `CONFIG_INVALID` when the first line is
+     * anything other than a version 1 header, or when the named index exists
+     * with a different schema.
      */
-    restore(indexName: string, data: Uint8Array): Promise<void>;
+    importDocumentExport(indexName: string, data: ByteSource): Promise<DocumentImportResult>;
     /**
      * Writes an index's current state to the durability directory, so a restart
      * replays less of the log.
@@ -599,4 +508,4 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations {
     shutdown(): Promise<void>;
 }
 
-export type { DocumentWriteOperations as D, IndexLifecycleOperations as I, NarsilEventMap as N, Narsil as a };
+export type { ByteSource as B, DocumentWriteOperations as D, IndexLifecycleOperations as I, Narsil as N };