@infino-ai/infino 0.9.0 → 0.9.2

package/infino/index.d.ts

@@ -19,6 +19,60 @@ export type SchemaDescriptor = Record<string, string | {
 }>;
 /** Accepted shapes for `Table.append`. */
 export type AppendData = RowRecord[] | arrow.Table | arrow.RecordBatch | Buffer | Uint8Array;
+/** A column's index in the schema document: full-text or vector. */
+export type ColumnIndex = {
+    kind: "fts";
+    analyzer: string;
+    stopwords?: string;
+    stemmer?: string;
+    positions: boolean;
+    stored: boolean;
+    k1: number;
+    b: number;
+} | {
+    kind: "vector";
+    metric: Metric;
+    rot_seed: string;
+    rerank_codec: string;
+};
+/** One field of the schema document. `type` is the document's type vocabulary (`"i64"`, `"large_utf8"`, …); a vector column carries `dim`. */
+export interface SchemaField {
+    id: number;
+    name: string;
+    type: string;
+    dim?: number;
+    nullable: boolean;
+    index?: ColumnIndex;
+    converting_from?: {
+        type: string;
+        dim?: number;
+    };
+}
+/** The schema document `Connection.schema` returns. */
+export interface TableSchema {
+    schema_id: number;
+    last_field_id: number;
+    max_fields: number;
+    max_depth: number;
+    fields: SchemaField[];
+    tombstoned: number[];
+}
+/** One field of a schema patch: a new column (`type` required), or a change to a live one addressed by `id` or `name`. */
+export interface SchemaFieldPatch {
+    id?: number;
+    name: string;
+    type?: string;
+    dim?: number;
+    nullable?: boolean;
+    index?: ColumnIndex;
+    dropped?: boolean;
+}
+/** What `Connection.schema(name, patch)` accepts: the document's own shape, every key optional. */
+export interface SchemaPatch {
+    fields?: SchemaFieldPatch[];
+    max_fields?: number;
+    max_depth?: number;
+}
 /**
  * Storage and cache config the `connect` URI can't carry. All optional.
  *
@@ -138,6 +192,9 @@ export interface ReindexReport {
     /** Columns whose text was never stored, so no repair can fix their terms;
      * only re-ingesting them from source can. */
     unrepairableColumns: string[];
+    /** Superfiles left untouched because their footer places a blob where
+     * the file or its manifest entry contradicts; they need inspection. */
+    inconsistentFooters: string[];
 }
 /** What a `reindex` would do, and what it would cost. */
 export interface StalenessReport {
@@ -152,6 +209,9 @@ export interface StalenessReport {
     bytesToRewrite: number;
     /** Columns no repair can fix, because their text was never stored. */
     unrepairableColumns: string[];
+    /** Superfiles whose footer places a blob where the file or its manifest
+     * entry contradicts. A reindex reports these and never rewrites them. */
+    inconsistentFooters: string[];
     /** Whether a reindex would do nothing at all. */
     isCurrent: boolean;
 }
@@ -334,6 +394,17 @@ export declare class Connection {
     /** Create a table from an apache-arrow `Schema` or `{ column: type }`. */
     createTable(name: string, schema: arrow.Schema | SchemaDescriptor | Buffer, indexes: IndexSpec): Table;
     openTable(name: string): Table;
+    /**
+     * The schema document of `name`: its fields with ids, types, nullability
+     * and indexes, the field cap and the `schema_id`. With `patch` (the same
+     * shape), merge it into the schema — or create the table from it when
+     * there is none — and return the document afterwards. A field is matched
+     * by `id` when it carries one and by `name` otherwise; an unmatched field
+     * is added, a different type changes the column, `dropped: true` retires
+     * it, and a field not mentioned is untouched. `expectedSchemaId` is a
+     * compare-and-set against the current `schema_id`.
+     */
+    schema(name: string, patch?: SchemaPatch, expectedSchemaId?: number): TableSchema;
     /** Drop a table. `purge` defaults to `true`, which also deletes the table's storage; pass `false` to only unregister it and keep the bytes. */
     dropTable(name: string, purge?: boolean): void;
     listTables(): string[];

package/infino/index.js

@@ -134,33 +134,58 @@ function buildColumn(field, rows) {
     }
     return arrow.vectorFromArray(values, field.type);
 }
-// Normalize append input -> IPC bytes. An array of objects, or an
-// apache-arrow Table / RecordBatch (normalized to rows via its own
-// `toArray()`/`toJSON()`); either way the columns are rebuilt in our arrow
-// instance from the declared schema. (We can't feed the consumer's Table
-// straight into our `tableToIPC` — a different module instance isn't
-// recognized.)
-function dataToIpc(data, getSchema) {
+// A `bigint` has no JSON form, so it travels as a one-key object the
+// replacer alone can produce and is then spliced back to the integer
+// literal it is — exact, and an integer rather than a float or a string.
+// A string in the data cannot be mistaken for one: `JSON.stringify`
+// escapes the quotes of a user's string, and the pattern below matches
+// only unescaped ones.
+const BIGINT_KEY = "$infino$bigint";
+function rowsToJson(rows) {
+    const text = JSON.stringify(rows, (_key, value) => typeof value === "bigint" ? { [BIGINT_KEY]: value.toString() } : value);
+    return text.replace(/\{"\$infino\$bigint":"(-?\d+)"\}/g, "$1");
+}
+// Values JSON has no faithful spelling for. A `Buffer` stringifies to
+// `{"type":"Buffer","data":[...]}` and a `Date` to a string, so neither
+// throws on the document path and neither arrives as what it was. Rows
+// carrying them are refused with the route that does carry them, rather
+// than written as the object or the string JSON turned them into.
+function unspellableValue(rows) {
+    for (const row of rows) {
+        for (const value of Object.values(row)) {
+            if (typeof Buffer !== "undefined" && Buffer.isBuffer(value))
+                return "a Buffer";
+            if (value instanceof Uint8Array)
+                return "a Uint8Array";
+            if (value instanceof Date)
+                return "a Date";
+        }
+    }
+    return undefined;
+}
+function writeInput(data) {
     if (Buffer.isBuffer(data))
-        return data;
+        return { ipc: data };
     if (data instanceof Uint8Array)
-        return Buffer.from(data);
-    let rows;
-    const d = data;
+        return { ipc: Buffer.from(data) };
     if (Array.isArray(data)) {
-        rows = data;
-    }
-    else if (d && (Array.isArray(d.batches) || (d.schema && typeof d.numRows === "number"))) {
-        rows = Array.from(d).map((r) => r.toJSON());
-    }
-    else {
-        throw new TypeError("append: expected an array of objects, an apache-arrow Table / RecordBatch, or an Arrow IPC Buffer");
+        const unspellable = unspellableValue(data);
+        if (unspellable) {
+            throw new TypeError(`append: ${unspellable} has no document form and would be written as what JSON ` +
+                "turned it into; pass an apache-arrow Table or RecordBatch to keep its type");
+        }
+        return { rows: rowsToJson(data) };
     }
-    const schema = getSchema();
-    const cols = {};
-    for (const field of schema.fields)
-        cols[field.name] = buildColumn(field, rows);
-    return Buffer.from(arrow.tableToIPC(new arrow.Table(cols), STREAM));
+    const d = data;
+    if (d && (Array.isArray(d.batches) || (d.schema && typeof d.numRows === "number"))) {
+        const rows = Array.from(d).map((r) => r.toJSON());
+        const fields = d.schema.fields.map((f) => new arrow.Field(f.name, nativeTypeFromForeign(f.type), f.nullable));
+        const cols = {};
+        for (const field of fields)
+            cols[field.name] = buildColumn(field, rows);
+        return { ipc: Buffer.from(arrow.tableToIPC(new arrow.Table(cols), STREAM)) };
+    }
+    throw new TypeError("append: expected an array of objects, an apache-arrow Table / RecordBatch, or an Arrow IPC Buffer");
 }
 // A Decimal128 value renders as a 4×u32 little-endian array in records.
 // Convert (scale-0 -> integer) to a `bigint`, matching token/exact match.
@@ -247,8 +272,8 @@ class Table {
      * append == one commit.
      */
     append(data) {
-        const ipc = dataToIpc(data, () => this.schema());
-        guard(this.remote, () => this.inner.append(ipc));
+        const input = writeInput(data);
+        guard(this.remote, () => ("rows" in input ? this.inner.appendRows(input.rows) : this.inner.append(input.ipc)));
     }
     /**
      * {@link append}, naming the source the rows came from. The superfiles
@@ -259,8 +284,8 @@ class Table {
      * a hosted table.
      */
     appendNamed(data, sourceName) {
-        const ipc = dataToIpc(data, () => this.schema());
-        guard(this.remote, () => this.inner.appendNamed(ipc, sourceName));
+        const input = writeInput(data);
+        guard(this.remote, () => "rows" in input ? this.inner.appendRowsNamed(input.rows, sourceName) : this.inner.appendNamed(input.ipc, sourceName));
     }
     bm25Search(column, query, k, opts = {}) {
         const buf = guard(this.remote, () => this.inner.bm25Search(column, query, k, opts.mode, opts.stats, opts.projection, opts.k1, opts.b));
@@ -293,8 +318,8 @@ class Table {
      * `data` (same shapes as `append`), 1:1 — the matched count must equal the
      * replacement-row count. Requires durable storage (not `memory://`). */
     update(predicate, data) {
-        const ipc = dataToIpc(data, () => this.schema());
-        return guard(this.remote, () => this.inner.update(predicate, ipc));
+        const input = writeInput(data);
+        return guard(this.remote, () => "rows" in input ? this.inner.updateRows(predicate, input.rows) : this.inner.update(predicate, input.ipc));
     }
     /** Delete rows matching a SQL predicate (e.g. `"status = 'spam'"`).
      * Requires durable storage (not `memory://`). */
@@ -373,6 +398,20 @@ class Connection {
     openTable(name) {
         return new Table(guard(this.remote, () => this.inner.openTable(name)), this.remote);
     }
+    /**
+     * The schema document of `name`: its fields with ids, types, nullability
+     * and indexes, the field cap and the `schema_id`. With `patch` (the same
+     * shape), merge it into the schema — or create the table from it when
+     * there is none — and return the document afterwards. A field is matched
+     * by `id` when it carries one and by `name` otherwise; an unmatched field
+     * is added, a different type changes the column, `dropped: true` retires
+     * it, and a field not mentioned is untouched. `expectedSchemaId` is a
+     * compare-and-set against the current `schema_id`.
+     */
+    schema(name, patch, expectedSchemaId) {
+        const text = guard(this.remote, () => this.inner.schema(name, patch === undefined ? undefined : JSON.stringify(patch), expectedSchemaId));
+        return JSON.parse(text);
+    }
     /** Drop a table. `purge` defaults to `true`, which also deletes the table's storage; pass `false` to only unregister it and keep the bytes. */
     dropTable(name, purge) {
         guard(this.remote, () => this.inner.dropTable(name, purge));

package/infino/native.d.ts

@@ -130,6 +130,11 @@ export interface ReindexReport {
    * only re-ingesting them from source can.
    */
   unrepairableColumns: Array<string>
+  /**
+   * Superfiles left untouched because their footer places a blob where
+   * the file or its manifest entry contradicts; they need inspection.
+   */
+  inconsistentFooters: Array<string>
 }
 /** What a `reindex` would do, and what it would cost. */
 export interface StalenessReport {
@@ -146,6 +151,11 @@ export interface StalenessReport {
   bytesToRewrite: number
   /** Columns no repair can fix, because their text was never stored. */
   unrepairableColumns: Array<string>
+  /**
+   * Superfiles whose footer places a blob where the file or its manifest
+   * entry contradicts. A reindex reports these and never rewrites them.
+   */
+  inconsistentFooters: Array<string>
   /** Whether a reindex would do nothing at all. */
   isCurrent: boolean
 }
@@ -290,6 +300,14 @@ export declare class Connection {
   createTable(name: string, schema: Buffer, indexes: IndexSpec): Table
   /** Open an existing table by name. */
   openTable(name: string): Table
+  /**
+   * The table's schema document as JSON text, or — with `patch_json`, a
+   * patch in the same shape — the document after merging the patch into
+   * the schema (creating the table when there is none). The JS wrapper
+   * parses the result and stringifies the patch; `expected_schema_id` is
+   * a compare-and-set against the current `schema_id`.
+   */
+  schema(name: string, patchJson?: string | undefined | null, expectedSchemaId?: number | undefined | null): string
   /**
    * Drop a table. `purge` defaults to `true`, which also deletes the
    * table's storage subtree after the catalog commit, reclaiming the bytes;
@@ -303,6 +321,7 @@ export declare class Connection {
    * the JS side reads with `tableFromIPC`. Search is available in SQL
    * via the TVFs, e.g.
    * `SELECT _id, score FROM bm25_search('docs', 'body', 'q', 10)`.
+   * Read-only, one statement per call: a write throws `InvalidArg`.
    */
   querySql(sql: string): Buffer
 }
@@ -315,6 +334,24 @@ export declare class Table {
    * concatenated into one commit; an empty stream is a no-op.
    */
   append(data: Buffer): void
+  /**
+   * Append rows given as a JSON array of objects (the text the JS wrapper
+   * produces with `JSON.stringify`): nested objects flatten to dot paths,
+   * arrays become list columns, a key the table has not seen adds a
+   * column typed from its values, and a value whose type disagrees with
+   * the column's is refused.
+   */
+  appendRows(rowsJson: string): void
+  /**
+   * `appendRows`, naming the source the rows came from as `appendNamed`
+   * does.
+   */
+  appendRowsNamed(rowsJson: string, sourceName: string): void
+  /**
+   * `update` with the replacement rows as a JSON array of objects, mapped
+   * as `appendRows` maps them.
+   */
+  updateRows(predicate: string, rowsJson: string): MutationStats
   /**
    * `append`, naming the source the rows came from: the superfiles this
    * commit writes are keyed `data/<stem>-<uuid>.sf.parquet`, with the stem

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@infino-ai/infino",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Fast search on object storage — SQL, full-text, and vector search.",
   "license": "Apache-2.0",
   "publishConfig": {
@@ -82,12 +82,12 @@
     "apache-arrow": "^17"
   },
   "optionalDependencies": {
-    "infx-darwin-x64": "0.9.0",
-    "infx-darwin-arm64": "0.9.0",
-    "infx-linux-x64-gnu": "0.9.0",
-    "infx-linux-arm64-gnu": "0.9.0",
-    "infx-linux-x64-musl": "0.9.0",
-    "infx-linux-arm64-musl": "0.9.0"
+    "infx-darwin-x64": "0.9.2",
+    "infx-darwin-arm64": "0.9.2",
+    "infx-linux-x64-gnu": "0.9.2",
+    "infx-linux-arm64-gnu": "0.9.2",
+    "infx-linux-x64-musl": "0.9.2",
+    "infx-linux-arm64-musl": "0.9.2"
   },
   "devDependencies": {
     "@napi-rs/cli": "^2.18.0",