dbgate-sqlite-dumper 0.1.0 → 0.1.1

package/dist/index.cjs

@@ -26,6 +26,7 @@ __export(src_exports, {
   DEFENSIVE_WARNING_COMMENT: () => DEFENSIVE_WARNING_COMMENT,
   InvalidTextEncodingError: () => InvalidTextEncodingError,
   MalformedSqlDumpError: () => MalformedSqlDumpError,
+  NATIVE_VALUE_TRANSPORT: () => NATIVE_VALUE_TRANSPORT,
   OperationCancelledError: () => OperationCancelledError,
   RestoreError: () => RestoreError,
   RestoreExecutionError: () => RestoreExecutionError,
@@ -51,6 +52,7 @@ __export(src_exports, {
   classifyWord: () => classifyWord,
   columnValueSelect: () => columnValueSelect,
   completeTransition: () => completeTransition,
+  connectionFeatures: () => connectionFeatures,
   createArchiveIdentity: () => createArchiveIdentity,
   createCanonicalIdentity: () => createCanonicalIdentity,
   createDumpId: () => createDumpId,
@@ -117,10 +119,12 @@ __export(src_exports, {
   throwIfAborted: () => throwIfAborted2,
   toAsciiLowerCase: () => toAsciiLowerCase,
   toAsciiUpperCase: () => toAsciiUpperCase,
+  toBuffer: () => toBuffer,
   toByteStringChunks: () => toByteStringChunks,
   toNumber: () => toNumber,
   toText: () => toText,
-  unsupportedFeatureDiagnostics: () => unsupportedFeatureDiagnostics
+  unsupportedFeatureDiagnostics: () => unsupportedFeatureDiagnostics,
+  valueTransport: () => valueTransport
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -167,19 +171,46 @@ function toText(value) {
   if (typeof value === "string") {
     return value;
   }
-  if (value instanceof Uint8Array) {
-    return Buffer.from(value).toString("utf8");
+  if (value instanceof Uint8Array || value instanceof ArrayBuffer || Array.isArray(value)) {
+    return toBuffer(value).toString("utf8");
   }
   return String(value);
 }
+function toBuffer(value) {
+  if (Buffer.isBuffer(value)) {
+    return value;
+  }
+  if (value instanceof Uint8Array) {
+    return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+  }
+  if (value instanceof ArrayBuffer) {
+    return Buffer.from(value);
+  }
+  return Buffer.from(value);
+}
+function connectionFeatures(connection) {
+  const features = connection.features ?? {};
+  const pageSize = features.pagedReadSize;
+  return {
+    transactions: features.transactions ?? true,
+    pragmaFunctions: features.pragmaFunctions ?? true,
+    extendedPragmas: features.extendedPragmas ?? true,
+    schemaQualifiedNames: features.schemaQualifiedNames ?? true,
+    binaryTransport: features.binaryTransport ?? "native",
+    pagedReadSize: pageSize === void 0 || !Number.isFinite(pageSize) ? void 0 : Math.max(1, Math.floor(pageSize)),
+    reservedNamePrefixes: features.reservedNamePrefixes ?? []
+  };
+}
 
 // src/connection/session.ts
 var DUMP_SAVEPOINT = "dbgate_sqlite_dump";
 async function beginSqliteDumpSession(connection, options, signal) {
-  const consistency = options?.consistency ?? "snapshot";
+  const requested = options?.consistency ?? "snapshot";
   const joinedExistingTransaction = connection.isInTransaction?.() ?? false;
+  const snapshotUnavailable = requested === "snapshot" && !connectionFeatures(connection).transactions;
+  const consistency = snapshotUnavailable ? "none" : requested;
   if (consistency === "none") {
-    return { consistency, joinedExistingTransaction, finish: async () => {
+    return { consistency, joinedExistingTransaction, snapshotUnavailable, finish: async () => {
     } };
   }
   await executeStatement(connection, `SAVEPOINT ${DUMP_SAVEPOINT}`, signal);
@@ -198,7 +229,7 @@ async function beginSqliteDumpSession(connection, options, signal) {
     await finish();
     throw error;
   }
-  return { consistency, joinedExistingTransaction, finish };
+  return { consistency, joinedExistingTransaction, snapshotUnavailable: false, finish };
 }
 
 // src/version/types.ts
@@ -1024,17 +1055,34 @@ async function introspectSqlite(connection, options, signal) {
   const schemaName = options?.schemaName ?? "main";
   const version = await detectSqliteVersion(connection, signal);
   const capabilities = detectSqliteCapabilities(version);
+  const features = connectionFeatures(connection);
   const diagnostics = [];
-  const catalog = new CatalogReader(connection, schemaName, capabilities, signal);
-  const schemaRows = await catalog.schemaRows();
-  if (schemaRows === null) {
+  if (!features.schemaQualifiedNames && toAsciiLowerCase(schemaName) !== "main") {
+    throw new SqliteDumperError(
+      "schema-not-found",
+      `This connection can only read the main database, not ${JSON.stringify(schemaName)}`
+    );
+  }
+  const catalog = new CatalogReader(connection, schemaName, capabilities, features, signal);
+  const allSchemaRows = await catalog.schemaRows();
+  if (allSchemaRows === null) {
     throw new SqliteDumperError(
       "schema-not-found",
       `No database named ${JSON.stringify(schemaName)} is attached to this connection`
     );
   }
+  const reserved = allSchemaRows.filter((row) => isReservedRow(row, features.reservedNamePrefixes));
+  const schemaRows = allSchemaRows.filter((row) => !reserved.includes(row));
+  if (reserved.length > 0) {
+    diagnostics.push({
+      severity: "info",
+      code: "reserved-objects-skipped",
+      message: `Left out ${reserved.length} object${reserved.length === 1 ? "" : "s"} the database engine reserves for itself and does not let clients read: ${reserved.map((row) => JSON.stringify(row.name)).join(", ")}.`
+    });
+  }
   const tableList = await catalog.tableList();
   const virtualTableNames = schemaRows.filter((row) => row.type === "table" && isVirtualTableSql(row.sql)).map((row) => row.name);
+  await catalog.prefetch(schemaRows);
   const tables = [];
   const indexes = [];
   const foreignKeys = [];
@@ -1135,6 +1183,20 @@ async function introspectSqlite(connection, options, signal) {
 function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
+function isReservedRow(row, prefixes) {
+  if (prefixes.length === 0) {
+    return false;
+  }
+  const names = [toAsciiLowerCase(row.name), toAsciiLowerCase(row.tableName)];
+  return prefixes.some((prefix) => {
+    const lower = toAsciiLowerCase(prefix);
+    return names.some((name) => name.startsWith(lower));
+  });
+}
+var PREFETCH_BATCH_SIZE = 50;
+function queryKey(query) {
+  return `${query.sql}\0${JSON.stringify(query.parameters ?? [])}`;
+}
 function isVirtualTableSql(sql) {
   return sql !== null && sql.startsWith("CREATE VIRTUAL TABLE");
 }
@@ -1178,43 +1240,142 @@ function findRowidAlias(columns, indexes) {
   return column.name;
 }
 var CatalogReader = class {
-  constructor(connection, schemaName, capabilities, signal) {
+  constructor(connection, schemaName, capabilities, features, signal) {
     this.connection = connection;
     this.schemaName = schemaName;
     this.capabilities = capabilities;
+    this.features = features;
     this.signal = signal;
-    this.qualifier = quoteIdentifier(schemaName);
+    this.qualifier = features.schemaQualifiedNames ? `${quoteIdentifier(schemaName)}.` : "";
+    this.usePragmaFunctions = capabilities.supportsPragmaFunctions && features.pragmaFunctions;
+    this.useTableXinfo = capabilities.supportsTableXinfo && features.extendedPragmas;
+    this.useIndexXinfo = features.extendedPragmas;
   }
   connection;
   schemaName;
   capabilities;
+  features;
   signal;
+  /** `"schema".`, or empty for a connection that cannot qualify names. */
   qualifier;
+  usePragmaFunctions;
+  // Not readonly: an engine that refuses the extended pragmas outright (some
+  // remote engines allow only the classic ones) is switched to the classic
+  // ones the first time one of them succeeds where its extended form failed.
+  useTableXinfo;
+  useIndexXinfo;
+  prefetched = /* @__PURE__ */ new Map();
   async rows(sql, parameters) {
-    const result = await this.connection.query(
-      parameters === void 0 ? { sql } : { sql, parameters },
-      this.signal
-    );
+    return this.run(parameters === void 0 ? { sql } : { sql, parameters });
+  }
+  async run(query) {
+    const prefetched = this.prefetched.get(queryKey(query));
+    if (prefetched !== void 0) {
+      return prefetched;
+    }
+    const result = await this.connection.query(query, this.signal);
     return result.rows;
   }
-  /** Runs `PRAGMA schema.name(argument)`, as a table-valued function when the library has them. */
-  async pragma(name, argument, columns) {
-    if (this.capabilities.supportsPragmaFunctions) {
-      return this.rows(`SELECT ${columns} FROM pragma_${name}(?, ?)`, [argument, this.schemaName]);
+  /** `PRAGMA schema.name(argument)`, as a table-valued function when the library has them. */
+  pragmaQuery(name, argument, columns) {
+    if (this.usePragmaFunctions) {
+      return {
+        sql: `SELECT ${columns} FROM pragma_${name}(?, ?)`,
+        parameters: [argument, this.schemaName]
+      };
     }
-    return this.rows(`PRAGMA ${this.qualifier}.${name}(${quoteStringLiteral(argument)})`);
+    return { sql: `PRAGMA ${this.qualifier}${name}(${quoteStringLiteral(argument)})` };
   }
-  async schemaRows() {
-    const databases = await this.rows("PRAGMA database_list");
-    const known = databases.some(
-      (row) => toAsciiLowerCase(toText(row.name) ?? "") === toAsciiLowerCase(this.schemaName)
+  columnsQuery(tableName) {
+    const withHidden = this.useTableXinfo;
+    return this.pragmaQuery(
+      withHidden ? "table_xinfo" : "table_info",
+      tableName,
+      `cid, name, type, "notnull" AS isNotNull, dflt_value AS defaultValue, pk${withHidden ? ", hidden" : ""}`
     );
-    if (!known) {
-      return null;
+  }
+  indexListQuery(tableName) {
+    return this.pragmaQuery(
+      "index_list",
+      tableName,
+      'seq, name, "unique" AS isUnique, origin, partial'
+    );
+  }
+  indexColumnsQuery(indexName) {
+    return this.useIndexXinfo ? this.pragmaQuery(
+      "index_xinfo",
+      indexName,
+      'seqno, cid, name, "desc" AS isDesc, coll, "key" AS isKey'
+    ) : this.pragmaQuery("index_info", indexName, "seqno, cid, name");
+  }
+  foreignKeysQuery(tableName) {
+    return this.pragmaQuery(
+      "foreign_key_list",
+      tableName,
+      'id, seq, "table" AS referencedTable, "from" AS fromColumn, "to" AS toColumn, on_update AS onUpdate, on_delete AS onDelete, "match" AS matchMode'
+    );
+  }
+  /**
+   * Fetches the per-table catalog of every object up front through
+   * {@link SqliteConnection.queryBatch}, when the connection has it — for a
+   * remote engine, a few requests instead of several per table. A batch that
+   * fails is simply not cached: its queries then run one by one later, where
+   * a failing one is handled (or reported) as it would be without batching.
+   */
+  async prefetch(schemaRows) {
+    if (!this.connection.queryBatch) {
+      return;
+    }
+    const queries = [];
+    for (const row of schemaRows) {
+      if (row.type === "table" && row.sql !== null) {
+        queries.push(this.columnsQuery(row.name));
+        if (!isVirtualTableSql(row.sql)) {
+          queries.push(this.indexListQuery(row.name), this.foreignKeysQuery(row.name));
+        }
+      } else if (row.type === "index") {
+        queries.push(this.indexColumnsQuery(row.name));
+      }
+    }
+    for (let start = 0; start < queries.length; start += PREFETCH_BATCH_SIZE) {
+      throwIfAborted2(this.signal);
+      const batch = queries.slice(start, start + PREFETCH_BATCH_SIZE);
+      let results;
+      try {
+        results = await this.connection.queryBatch(batch, this.signal);
+      } catch (error) {
+        if (isAbortError(error)) throw error;
+        continue;
+      }
+      if (results.length !== batch.length) {
+        continue;
+      }
+      batch.forEach((query, index) => {
+        this.prefetched.set(
+          queryKey(query),
+          results[index].rows
+        );
+      });
+    }
+  }
+  async schemaRows() {
+    if (this.features.schemaQualifiedNames) {
+      let databases;
+      try {
+        databases = await this.rows("PRAGMA database_list");
+      } catch (error) {
+        if (isAbortError(error) || toAsciiLowerCase(this.schemaName) !== "main") throw error;
+      }
+      const known = databases === void 0 || databases.some(
+        (row) => toAsciiLowerCase(toText(row.name) ?? "") === toAsciiLowerCase(this.schemaName)
+      );
+      if (!known) {
+        return null;
+      }
     }
     const master = toAsciiLowerCase(this.schemaName) === "temp" ? "sqlite_temp_master" : "sqlite_master";
     const rows = await this.rows(
-      `SELECT rowid AS schemaRowid, type, name, tbl_name AS tableName, sql FROM ${this.qualifier}.${master} ORDER BY rowid`
+      `SELECT rowid AS schemaRowid, type, name, tbl_name AS tableName, sql FROM ${this.qualifier}${master} ORDER BY rowid`
     );
     return rows.map((row) => ({
       schemaRowid: toNumber(row.schemaRowid),
@@ -1225,13 +1386,14 @@ var CatalogReader = class {
     }));
   }
   async tableList() {
-    if (!this.capabilities.supportsTableList) {
+    if (!this.capabilities.supportsTableList || !this.features.extendedPragmas) {
       return void 0;
     }
     try {
-      const rows = await this.rows(
-        "SELECT name, type, wr, strict FROM pragma_table_list WHERE schema = ?",
-        [this.schemaName]
+      const rows = this.usePragmaFunctions ? await this.rows("SELECT name, type, wr, strict FROM pragma_table_list WHERE schema = ?", [
+        this.schemaName
+      ]) : (await this.rows(`PRAGMA ${this.qualifier}table_list`)).filter(
+        (row) => !this.features.schemaQualifiedNames || toAsciiLowerCase(toText(row.schema) ?? "") === toAsciiLowerCase(this.schemaName)
       );
       return new Map(
         rows.map((row) => [
@@ -1248,13 +1410,27 @@ var CatalogReader = class {
       return void 0;
     }
   }
+  /** Runs an extended pragma's query, retrying it in its classic form if the engine refuses it. */
+  async runExtended(flag, query) {
+    try {
+      return await this.run(query());
+    } catch (error) {
+      if (isAbortError(error) || !this[flag]) throw error;
+      this[flag] = false;
+      try {
+        return await this.run(query());
+      } catch {
+        this[flag] = true;
+        throw error;
+      }
+    }
+  }
   async columns(tableName) {
-    const withHidden = this.capabilities.supportsTableXinfo;
-    const rows = await this.pragma(
-      withHidden ? "table_xinfo" : "table_info",
-      tableName,
-      `cid, name, type, "notnull" AS isNotNull, dflt_value AS defaultValue, pk${withHidden ? ", hidden" : ""}`
-    );
+    let withHidden = this.useTableXinfo;
+    const rows = await this.runExtended("useTableXinfo", () => {
+      withHidden = this.useTableXinfo;
+      return this.columnsQuery(tableName);
+    });
     return rows.map((row) => {
       const hidden = withHidden ? toNumber(row.hidden) : 0;
       return {
@@ -1270,11 +1446,7 @@ var CatalogReader = class {
     });
   }
   async indexes(tableName, schemaRows) {
-    const list = await this.pragma(
-      "index_list",
-      tableName,
-      'seq, name, "unique" AS isUnique, origin, partial'
-    );
+    const list = await this.run(this.indexListQuery(tableName));
     const result = [];
     for (const row of list) {
       const name = toText(row.name) ?? "";
@@ -1282,12 +1454,11 @@ var CatalogReader = class {
       const origin = toText(row.origin) ?? "c";
       let columns = [];
       try {
-        const keyRows = await this.pragma(
-          "index_xinfo",
-          name,
-          'seqno, cid, name, "desc" AS isDesc, coll, "key" AS isKey'
-        );
-        columns = keyRows.filter((keyRow) => toNumber(keyRow.isKey ?? keyRow.key) !== 0).map((keyRow) => ({
+        const keyRows = await this.runExtended("useIndexXinfo", () => this.indexColumnsQuery(name));
+        columns = keyRows.filter((keyRow) => {
+          const isKey = keyRow.isKey ?? keyRow.key;
+          return isKey === void 0 || toNumber(isKey) !== 0;
+        }).sort((a, b) => toNumber(a.seqno) - toNumber(b.seqno)).map((keyRow) => ({
           name: toText(keyRow.name),
           cid: toNumber(keyRow.cid),
           descending: toNumber(keyRow.isDesc ?? keyRow.desc) !== 0,
@@ -1312,11 +1483,7 @@ var CatalogReader = class {
     return result;
   }
   async foreignKeys(tableName) {
-    const rows = await this.pragma(
-      "foreign_key_list",
-      tableName,
-      'id, seq, "table" AS referencedTable, "from" AS fromColumn, "to" AS toColumn, on_update AS onUpdate, on_delete AS onDelete, "match" AS matchMode'
-    );
+    const rows = await this.run(this.foreignKeysQuery(tableName));
     const byId = /* @__PURE__ */ new Map();
     const ordered = [...rows].sort(
       (a, b) => toNumber(a.id) - toNumber(b.id) || toNumber(a.seq) - toNumber(b.seq)
@@ -1345,7 +1512,7 @@ var CatalogReader = class {
   }
   async sequences() {
     const rows = await this.rows(
-      `SELECT name, CAST(seq AS TEXT) AS value FROM ${this.qualifier}.sqlite_sequence`
+      `SELECT name, CAST(seq AS TEXT) AS value FROM ${this.qualifier}sqlite_sequence`
     );
     return rows.map((row) => ({
       tableName: toText(row.name) ?? "",
@@ -1355,7 +1522,7 @@ var CatalogReader = class {
   async settings() {
     const read = async (pragma) => {
       try {
-        const rows = await this.rows(`PRAGMA ${this.qualifier}.${pragma}`);
+        const rows = await this.rows(`PRAGMA ${this.qualifier}${pragma}`);
         const first = rows[0];
         return first === void 0 ? void 0 : Object.values(first)[0];
       } catch (error) {
@@ -1904,20 +2071,25 @@ var import_node_buffer = require("buffer");
 function realLiteralExpression(column) {
   return `CASE WHEN ${column} = 9e999 THEN '9.0e+999' WHEN ${column} = -9e999 THEN '-9.0e+999' WHEN ${column} >= -9223372036854775808.0 AND ${column} < 9223372036854775808.0 AND ${column} = CAST(${column} AS INTEGER) THEN CAST(CAST(${column} AS INTEGER) AS TEXT) || '.0' ELSE printf('%!.20g', ${column}) END`;
 }
-function columnValueSelect(column, index, utf8Database) {
-  const textBranch = utf8Database ? `CAST(${column} AS BLOB)` : column;
-  return `typeof(${column}) AS "t${index}", CASE typeof(${column}) WHEN 'integer' THEN CAST(${column} AS TEXT) WHEN 'real' THEN ${realLiteralExpression(column)} WHEN 'text' THEN ${textBranch} ELSE ${column} END AS "v${index}"`;
+var NATIVE_VALUE_TRANSPORT = { hexText: false, hexBlob: false };
+function valueTransport(utf8Database, binaryTransport) {
+  return binaryTransport === "hex" ? { hexText: utf8Database, hexBlob: true } : NATIVE_VALUE_TRANSPORT;
 }
-function asBytes(value) {
-  if (Buffer.isBuffer(value)) {
-    return value;
+function columnValueSelect(column, index, utf8Database, transport = NATIVE_VALUE_TRANSPORT) {
+  const textBytes = utf8Database ? `CAST(${column} AS BLOB)` : column;
+  const textBranch = transport.hexText ? `hex(${textBytes})` : textBytes;
+  return `typeof(${column}) AS "t${index}", CASE typeof(${column}) WHEN 'integer' THEN CAST(${column} AS TEXT) WHEN 'real' THEN ${realLiteralExpression(column)} WHEN 'text' THEN ${textBranch} ` + (transport.hexBlob ? `WHEN 'blob' THEN hex(${column}) ` : "") + `ELSE ${column} END AS "v${index}"`;
+}
+function asBytes(value, hex) {
+  if (typeof value === "string") {
+    return Buffer.from(value, hex ? "hex" : "utf8");
   }
-  if (value instanceof Uint8Array) {
-    return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+  if (value instanceof Uint8Array || value instanceof ArrayBuffer || Array.isArray(value)) {
+    return toBuffer(value);
   }
   return Buffer.from(String(value), "utf8");
 }
-function renderColumnLiteral(row, index, textStyle) {
+function renderColumnLiteral(row, index, textStyle, transport = NATIVE_VALUE_TRANSPORT) {
   const storageClass = row[`t${index}`];
   const value = row[`v${index}`] ?? null;
   switch (storageClass) {
@@ -1925,17 +2097,17 @@ function renderColumnLiteral(row, index, textStyle) {
     case "real":
       return String(value);
     case "text": {
-      if (typeof value === "string") {
+      if (typeof value === "string" && !transport.hexText) {
         return renderTextLiteral(value, textStyle);
       }
-      const bytes = asBytes(value);
+      const bytes = asBytes(value, transport.hexText);
       if ((0, import_node_buffer.isUtf8)(bytes)) {
         return renderTextLiteral(bytes.toString("utf8"), textStyle);
       }
       return Buffer.from(renderTextLiteral(bytes.toString("latin1"), textStyle), "latin1");
     }
     case "blob":
-      return renderBlobLiteral(asBytes(value));
+      return renderBlobLiteral(asBytes(value, transport.hexBlob));
     default:
       return "NULL";
   }
@@ -1972,6 +2144,8 @@ async function exportTableDataAsInserts(request) {
   );
   const textStyle = options.rawNewlines ? "raw-newlines" : "escaped";
   const utf8Database = (request.encoding ?? "UTF-8").toUpperCase() === "UTF-8";
+  const features = connectionFeatures(connection);
+  const transport = valueTransport(utf8Database, features.binaryTransport);
   const warnings = [];
   const columns = insertableColumns(table);
   const rowidName = options.preserveRowids ? preservableRowidName(table) : null;
@@ -2003,13 +2177,39 @@ async function exportTableDataAsInserts(request) {
   const columnList = rowidName ? `(${[rowidName, ...columns.map((column) => quoteIdentifierIfNeeded(column.name))].join(",")})` : "";
   const statementPrefix = `INSERT INTO ${quoteIdentifierIfNeeded(table.name)}${columnList} VALUES`;
   const selectList = [
-    ...rowidName ? [columnValueSelect(rowidName, 0, utf8Database)] : [],
+    ...rowidName ? [columnValueSelect(rowidName, 0, utf8Database, transport)] : [],
     ...columns.map(
-      (column, index) => columnValueSelect(quoteIdentifier(column.name), index + (rowidName ? 1 : 0), utf8Database)
+      (column, index) => columnValueSelect(
+        quoteIdentifier(column.name),
+        index + (rowidName ? 1 : 0),
+        utf8Database,
+        transport
+      )
     )
   ].join(", ");
   const valueCount = columns.length + (rowidName ? 1 : 0);
-  const sql = `SELECT ${selectList} FROM ${quoteQualifiedIdentifier(schemaName, table.name)}${systemRowFilterClause(table, request.systemRowFilter)}`;
+  const from = features.schemaQualifiedNames ? quoteQualifiedIdentifier(schemaName, table.name) : quoteIdentifier(table.name);
+  const filter = systemRowFilterCondition(
+    table,
+    request.systemRowFilter,
+    features.reservedNamePrefixes
+  );
+  const rows = features.pagedReadSize === void 0 ? connection.stream(
+    { sql: `SELECT ${selectList} FROM ${from}${filter === null ? "" : ` WHERE ${filter}`}` },
+    {
+      ...signal === void 0 ? {} : { signal },
+      ...options.streamBatchSize === void 0 ? {} : { batchSize: options.streamBatchSize }
+    }
+  ) : readPages({
+    connection,
+    selectList,
+    from,
+    filter,
+    pageSize: features.pagedReadSize,
+    key: pageKey(table, columns, rowidName ? 1 : 0, request.primaryKeyIndex),
+    transport,
+    signal
+  });
   let rowsExported = 0;
   let statementsWritten = 0;
   let output = new SqlChunkBuilder();
@@ -2050,13 +2250,7 @@ async function exportTableDataAsInserts(request) {
   };
   progress("started");
   try {
-    for await (const row of connection.stream(
-      { sql },
-      {
-        ...signal === void 0 ? {} : { signal },
-        ...options.streamBatchSize === void 0 ? {} : { batchSize: options.streamBatchSize }
-      }
-    )) {
+    for await (const row of rows) {
       throwIfAborted2(signal);
       const tuple = new SqlChunkBuilder();
       tuple.append("(");
@@ -2064,7 +2258,7 @@ async function exportTableDataAsInserts(request) {
         if (index > 0) {
           tuple.append(",");
         }
-        tuple.append(renderColumnLiteral(row, index, textStyle));
+        tuple.append(renderColumnLiteral(row, index, textStyle, transport));
       }
       tuple.append(")");
       if (statement && (statementRows >= maxRowsPerStatement || statement.length + tuple.length + 2 > maxStatementBytes)) {
@@ -2108,16 +2302,121 @@ async function exportTableDataAsInserts(request) {
     throw error;
   }
 }
-function systemRowFilterClause(table, filter) {
-  if (filter === void 0 || table.kind !== "system") {
-    return "";
+function systemRowFilterCondition(table, filter, reservedPrefixes) {
+  if (table.kind !== "system" || filter === void 0 && reservedPrefixes.length === 0) {
+    return null;
   }
   const column = table.name === "sqlite_sequence" ? "name" : "tbl";
-  const names = [...new Set(filter.map(toAsciiLowerCase))].map(quoteStringLiteral);
-  if (names.length === 0) {
-    return " WHERE 0";
+  const conditions = [];
+  if (filter !== void 0) {
+    const names = [...new Set(filter.map(toAsciiLowerCase))].map(quoteStringLiteral);
+    if (names.length === 0) {
+      return "0";
+    }
+    conditions.push(`lower(${column}) IN (${names.join(",")})`);
+  }
+  for (const prefix of reservedPrefixes) {
+    const lower = toAsciiLowerCase(prefix);
+    conditions.push(
+      `coalesce(substr(lower(${column}), 1, ${lower.length}) <> ${quoteStringLiteral(lower)}, 1)`
+    );
   }
-  return ` WHERE lower(${column}) IN (${names.join(",")})`;
+  return conditions.join(" AND ");
+}
+function pageKey(table, columns, valueOffset, primaryKeyIndex) {
+  if (table.kind === "virtual") {
+    return { kind: "offset" };
+  }
+  if (!table.withoutRowid) {
+    if (table.rowidAliasColumn !== null) {
+      return { kind: "rowid", expression: quoteIdentifier(table.rowidAliasColumn) };
+    }
+    const names = new Set(table.columns.map((column) => toAsciiLowerCase(column.name)));
+    const name = ROWID_NAMES.find((candidate) => !names.has(candidate));
+    return name === void 0 ? { kind: "offset" } : { kind: "rowid", expression: name };
+  }
+  const valueIndexes = new Map(
+    columns.map((column, index) => [toAsciiLowerCase(column.name), index + valueOffset])
+  );
+  const keyColumns = primaryKeyIndex && primaryKeyIndex.columns.length > 0 ? primaryKeyIndex.columns.map((column) => ({
+    name: column.name,
+    descending: column.descending,
+    collation: column.collation
+  })) : [...table.columns].filter((column) => column.primaryKeyPosition > 0).sort((a, b) => a.primaryKeyPosition - b.primaryKeyPosition).map((column) => ({ name: column.name, descending: false, collation: null }));
+  const result = [];
+  for (const column of keyColumns) {
+    const valueIndex = column.name === null ? void 0 : valueIndexes.get(toAsciiLowerCase(column.name));
+    if (column.name === null || valueIndex === void 0) {
+      return { kind: "offset" };
+    }
+    const collation = column.collation === null || toAsciiLowerCase(column.collation) === "binary" ? "" : ` COLLATE ${quoteIdentifier(column.collation)}`;
+    result.push({
+      expression: `${quoteIdentifier(column.name)}${collation}`,
+      valueIndex,
+      descending: column.descending
+    });
+  }
+  return result.length === 0 ? { kind: "offset" } : { kind: "primaryKey", columns: result };
+}
+var PAGE_KEY_ALIAS = "dbgate_page_key";
+async function* readPages(request) {
+  const { connection, selectList, from, filter, pageSize, key, transport, signal } = request;
+  let after = null;
+  let offset = 0;
+  for (; ; ) {
+    throwIfAborted2(signal);
+    const conditions = [filter, after].filter((condition) => condition !== null);
+    const where = conditions.length === 0 ? "" : ` WHERE ${conditions.map((c) => `(${c})`).join(" AND ")}`;
+    let sql;
+    if (key.kind === "rowid") {
+      sql = `SELECT ${selectList}, CAST(${key.expression} AS TEXT) AS "${PAGE_KEY_ALIAS}" FROM ${from}${where} ORDER BY ${key.expression} LIMIT ${pageSize}`;
+    } else if (key.kind === "primaryKey") {
+      const order = key.columns.map((column) => `${column.expression}${column.descending ? " DESC" : ""}`).join(", ");
+      sql = `SELECT ${selectList} FROM ${from}${where} ORDER BY ${order} LIMIT ${pageSize}`;
+    } else {
+      sql = `SELECT ${selectList} FROM ${from}${where} LIMIT ${pageSize} OFFSET ${offset}`;
+    }
+    const { rows } = await connection.query({ sql }, signal);
+    for (const row of rows) {
+      yield row;
+    }
+    if (rows.length < pageSize) {
+      return;
+    }
+    const last = rows[rows.length - 1];
+    if (key.kind === "rowid") {
+      after = `${key.expression} > ${String(last[PAGE_KEY_ALIAS])}`;
+    } else if (key.kind === "primaryKey") {
+      after = keyAfterCondition(key.columns, last, transport);
+    } else {
+      offset += rows.length;
+    }
+  }
+}
+function keyAfterCondition(columns, row, transport) {
+  const literals = columns.map((column) => keyLiteral(row, column.valueIndex, transport));
+  const alternatives = [];
+  for (let position = 0; position < columns.length; position++) {
+    const terms = [];
+    for (let previous = 0; previous < position; previous++) {
+      terms.push(
+        `${columns[previous].expression} = ${literals[previous]}`
+      );
+    }
+    const column = columns[position];
+    terms.push(`${column.expression} ${column.descending ? "<" : ">"} ${literals[position]}`);
+    alternatives.push(terms.length === 1 ? terms[0] : `(${terms.join(" AND ")})`);
+  }
+  return alternatives.join(" OR ");
+}
+function keyLiteral(row, index, transport) {
+  const literal = renderColumnLiteral(row, index, "escaped", transport);
+  if (typeof literal === "string") {
+    return literal;
+  }
+  const value = row[`v${index}`] ?? null;
+  const bytes = typeof value === "string" ? Buffer.from(value, transport.hexText ? "hex" : "utf8") : toBuffer(value);
+  return `CAST(X'${bytes.toString("hex").toUpperCase()}' AS TEXT)`;
 }
 
 // src/compatibility/check.ts
@@ -3726,6 +4025,9 @@ async function dumpSqlite(connectionInput, options, output, onProgress, signal)
       if (!table) {
         return false;
       }
+      const primaryKeyIndex = table.withoutRowid ? introspection.database.indexes.find(
+        (index) => index.origin === "pk" && foldSqliteName(index.tableName) === foldSqliteName(table.name)
+      ) : void 0;
       const result = await exportTableDataAsInserts({
         connection: acquired.connection,
         schemaName: introspection.database.schemaName,
@@ -3733,6 +4035,7 @@ async function dumpSqlite(connectionInput, options, output, onProgress, signal)
         writer,
         encoding: introspection.database.encoding,
         ...entry.systemRowFilter === void 0 ? {} : { systemRowFilter: entry.systemRowFilter },
+        ...primaryKeyIndex === void 0 ? {} : { primaryKeyIndex },
         ...options.dataExport === void 0 ? {} : { options: options.dataExport },
         ...signal === void 0 ? {} : { signal },
         ...onProgress === void 0 ? {} : { onProgress }
@@ -3757,13 +4060,21 @@ async function dumpSqlite(connectionInput, options, output, onProgress, signal)
       onTableData
     });
     onProgress?.({ phase: "finalizing", bytesWritten: renderResult.bytesWritten });
-    const sessionWarnings = session.joinedExistingTransaction ? [
-      {
+    const sessionWarnings = [];
+    if (session.joinedExistingTransaction) {
+      sessionWarnings.push({
         severity: "info",
         code: "dump-joined-caller-transaction",
         message: "The connection was already inside a transaction, so the dump was read within it and includes that transaction's uncommitted changes."
-      }
-    ] : [];
+      });
+    }
+    if (session.snapshotUnavailable) {
+      sessionWarnings.push({
+        severity: "warning",
+        code: "snapshot-unavailable",
+        message: "The database does not support transactions, so the dump was not read from one snapshot: changes committed while it ran may be partly included."
+      });
+    }
     return {
       bytesWritten: renderResult.bytesWritten,
       renderedDumpIds: renderResult.renderedDumpIds,
@@ -3803,6 +4114,7 @@ function dedupeDiagnostics(diagnostics) {
   DEFENSIVE_WARNING_COMMENT,
   InvalidTextEncodingError,
   MalformedSqlDumpError,
+  NATIVE_VALUE_TRANSPORT,
   OperationCancelledError,
   RestoreError,
   RestoreExecutionError,
@@ -3828,6 +4140,7 @@ function dedupeDiagnostics(diagnostics) {
   classifyWord,
   columnValueSelect,
   completeTransition,
+  connectionFeatures,
   createArchiveIdentity,
   createCanonicalIdentity,
   createDumpId,
@@ -3894,9 +4207,11 @@ function dedupeDiagnostics(diagnostics) {
   throwIfAborted,
   toAsciiLowerCase,
   toAsciiUpperCase,
+  toBuffer,
   toByteStringChunks,
   toNumber,
   toText,
-  unsupportedFeatureDiagnostics
+  unsupportedFeatureDiagnostics,
+  valueTransport
 });
 //# sourceMappingURL=index.cjs.map