@reddb-io/client 1.6.0 → 1.8.0

package/src/streaming.js

@@ -0,0 +1,362 @@
+/**
+ * Node-native streaming surface for the JS driver (PRD #759 / S11).
+ *
+ * Two transport-agnostic wrappers turn a low-level streaming *session*
+ * — supplied by whichever transport the connection uses (HTTP NDJSON or
+ * RedWire) — into idiomatic Node streams:
+ *
+ *   - `RowReadable`  — a `Readable` in object mode that also conforms to
+ *     `AsyncIterable<Row>` (Readable already is one). Rows flow with
+ *     natural backpressure via `read()` / `pause()` / `resume()`. The
+ *     descriptor and the resumable cursor (when the transport surfaces
+ *     them) arrive as `'descriptor'` / `'cursor'` events. A mid-stream
+ *     `error` frame surfaces both as an `'error'` event and as a
+ *     rejected `for await` iteration.
+ *   - `RowWritable`  — a `Writable` in object mode. Backpressure flows
+ *     through `write()`'s return value and `'drain'`. `end()` signals
+ *     end-of-stream; the server's terminal envelope resolves the
+ *     `.completion()` promise.
+ *
+ * Both expose a uniform `cancel(reason?)` that terminates the underlying
+ * transport stream (a `StreamCancel` frame over RedWire, an
+ * `AbortController.abort()` over HTTP) and rejects anything pending.
+ *
+ * The transport session contracts these wrappers consume:
+ *
+ *   read session (transport.streamSelect):
+ *     { [Symbol.asyncIterator](): AsyncIterator<{type,value}>,
+ *       cancel(reason): Promise<void> }
+ *     where `type` is 'descriptor' | 'cursor' | 'row' | 'end'; the
+ *     iterator throws a `RedDBError` when the server emits an error frame.
+ *
+ *   write session (transport.streamInput):
+ *     { write(row): Promise<void>,      // resolves when accepted (backpressure)
+ *       close(): Promise<EndEnvelope>,  // send terminal, await server end
+ *       cancel(reason): Promise<void> }
+ */
+
+import { Readable, Writable, Transform } from 'node:stream'
+import { RedDBError } from './protocol.js'
+import { classifyNdjsonFrame } from './core/ndjson.js'
+
+// `classifyNdjsonFrame` is a pure NDJSON helper that now lives in the
+// transport-agnostic core; re-exported here so the historical
+// `import { classifyNdjsonFrame } from './streaming.js'` path keeps working.
+export { classifyNdjsonFrame }
+
+function cancelError(reason) {
+  const suffix = typeof reason === 'string' && reason.length > 0 ? `: ${reason}` : ''
+  return new RedDBError('STREAM_CANCELLED', `stream cancelled${suffix}`)
+}
+
+/**
+ * `Readable` (object mode) over a transport read session. Also an
+ * `AsyncIterable<Row>` — `for await (const row of stream)` yields each
+ * row and exits cleanly on stream end.
+ */
+export class RowReadable extends Readable {
+  /**
+   * @param {Promise<object>} sessionPromise resolves to a read session.
+   * @param {{ signal?: AbortSignal }} [opts]
+   */
+  constructor(sessionPromise, { signal } = {}) {
+    super({ objectMode: true })
+    this._sessionPromise = sessionPromise
+    this._session = null
+    this._iter = null
+    this._pumping = false
+    this._ended = false
+    this._cancelReason = undefined
+    /** Schema descriptor (HTTP NDJSON) once seen; null otherwise. */
+    this.descriptor = null
+    /** Resumable cursor control frame (#807) once seen; null otherwise. */
+    this.cursor = null
+    /** Terminal `end` envelope once the stream completes; null otherwise. */
+    this.endInfo = null
+    if (signal) {
+      if (signal.aborted) {
+        queueMicrotask(() => this.cancel(abortReason(signal)))
+      } else {
+        signal.addEventListener('abort', () => this.cancel(abortReason(signal)), { once: true })
+      }
+    }
+  }
+
+  async _resolveIter() {
+    if (this._iter) return this._iter
+    this._session = await this._sessionPromise
+    this._iter = this._session[Symbol.asyncIterator]()
+    return this._iter
+  }
+
+  _read() {
+    if (this._pumping) return
+    this._pumping = true
+    this._pump().then(
+      () => {
+        this._pumping = false
+      },
+      (err) => {
+        this._pumping = false
+        this.destroy(err instanceof Error ? err : new RedDBError('STREAM_ERROR', String(err)))
+      },
+    )
+  }
+
+  async _pump() {
+    const iter = await this._resolveIter()
+    // Loop until backpressure (push returned false) or completion. Non-row
+    // control frames (descriptor/cursor) are surfaced as events and do not
+    // count against the readable buffer, so we keep pulling after them.
+    for (;;) {
+      const { value: frame, done } = await iter.next()
+      if (done) {
+        this._ended = true
+        this.push(null)
+        return
+      }
+      if (frame.type === 'descriptor') {
+        this.descriptor = frame.value
+        this.emit('descriptor', frame.value)
+        continue
+      }
+      if (frame.type === 'cursor') {
+        this.cursor = frame.value
+        this.emit('cursor', frame.value)
+        continue
+      }
+      if (frame.type === 'end') {
+        this.endInfo = frame.value
+        this._ended = true
+        this.push(null)
+        return
+      }
+      // frame.type === 'row'
+      if (!this.push(frame.value)) {
+        return
+      }
+    }
+  }
+
+  _destroy(err, callback) {
+    // A stream that ended on its own terminal frame needs no cancel — only
+    // an early teardown (error or explicit cancel) signals the transport.
+    const session = this._ended ? null : this._session
+    const reason = this._cancelReason
+    Promise.resolve(session && session.cancel ? session.cancel(reason) : undefined)
+      .catch(() => {})
+      .finally(() => callback(err))
+  }
+
+  /**
+   * Terminate the stream. Sends a transport-level cancel and rejects any
+   * pending `for await` iteration with a `STREAM_CANCELLED` error.
+   * @param {string} [reason]
+   * @returns {Promise<void>}
+   */
+  cancel(reason) {
+    if (this.destroyed) return Promise.resolve()
+    this._cancelReason = reason
+    return new Promise((resolve) => {
+      this.once('close', resolve)
+      this.destroy(cancelError(reason))
+    })
+  }
+}
+
+/**
+ * `Writable` (object mode) over a transport write session. End-of-stream
+ * is `end()`; the server's terminal envelope resolves `.completion()`.
+ */
+export class RowWritable extends Writable {
+  /**
+   * @param {Promise<object>} sessionPromise resolves to a write session.
+   * @param {{ signal?: AbortSignal }} [opts]
+   */
+  constructor(sessionPromise, { signal } = {}) {
+    super({ objectMode: true })
+    this._sessionPromise = sessionPromise
+    this._session = null
+    this._finished = false
+    this._cancelReason = undefined
+    this._completion = null
+    this._completionPromise = new Promise((resolve, reject) => {
+      this._completion = { resolve, reject }
+    })
+    // Don't crash the process if nobody attaches to completion() and the
+    // stream errors — the 'error' event already carries the failure.
+    this._completionPromise.catch(() => {})
+    if (signal) {
+      if (signal.aborted) {
+        queueMicrotask(() => this.cancel(abortReason(signal)))
+      } else {
+        signal.addEventListener('abort', () => this.cancel(abortReason(signal)), { once: true })
+      }
+    }
+  }
+
+  async _resolveSession() {
+    if (!this._session) this._session = await this._sessionPromise
+    return this._session
+  }
+
+  _write(row, _enc, callback) {
+    this._resolveSession()
+      .then((session) => session.write(row))
+      .then(() => callback(), (err) => callback(err))
+  }
+
+  _final(callback) {
+    this._resolveSession()
+      .then((session) => session.close())
+      .then(
+        (end) => {
+          this._finished = true
+          this._completion.resolve(end)
+          callback()
+        },
+        (err) => {
+          this._completion.reject(err)
+          callback(err)
+        },
+      )
+  }
+
+  _destroy(err, callback) {
+    if (err) this._completion.reject(err)
+    const session = this._finished ? null : this._session
+    const reason = this._cancelReason
+    Promise.resolve(session && session.cancel ? session.cancel(reason) : undefined)
+      .catch(() => {})
+      .finally(() => callback(err))
+  }
+
+  /**
+   * Resolves with the server's terminal `end` envelope once the stream
+   * finishes successfully; rejects if the stream errors or is cancelled.
+   * @returns {Promise<object>}
+   */
+  completion() {
+    return this._completionPromise
+  }
+
+  /**
+   * Terminate the stream without flushing the remaining rows. Rejects
+   * `.completion()` with a `STREAM_CANCELLED` error.
+   * @param {string} [reason]
+   * @returns {Promise<void>}
+   */
+  cancel(reason) {
+    if (this.destroyed) return Promise.resolve()
+    this._cancelReason = reason
+    return new Promise((resolve) => {
+      this.once('close', resolve)
+      this.destroy(cancelError(reason))
+    })
+  }
+}
+
+function abortReason(signal) {
+  const reason = signal?.reason
+  if (typeof reason === 'string') return reason
+  if (reason && typeof reason.message === 'string') return reason.message
+  return 'aborted'
+}
+
+/**
+ * A `Transform` that splits an NDJSON byte/text stream into parsed row
+ * objects, ready to pipe into `table.inputStream()`:
+ *
+ *   fs.createReadStream('rows.ndjson').pipe(splitNdjson()).pipe(table.inputStream())
+ *
+ * Each non-empty line is `JSON.parse`d; a `{ "row": {...} }` envelope is
+ * unwrapped to its inner object so files written by the streaming reader
+ * round-trip, while bare-object lines pass through untouched.
+ * @returns {Transform}
+ */
+export function splitNdjson() {
+  let buffer = ''
+  const parseLine = (line, push, callback) => {
+    const trimmed = line.trim()
+    if (trimmed.length === 0) return true
+    let parsed
+    try {
+      parsed = JSON.parse(trimmed)
+    } catch (err) {
+      callback(new RedDBError('NDJSON_PARSE_ERROR', `invalid NDJSON line: ${err.message}`))
+      return false
+    }
+    const row = parsed && typeof parsed === 'object' && 'row' in parsed ? parsed.row : parsed
+    push(row)
+    return true
+  }
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: false,
+    transform(chunk, _enc, callback) {
+      buffer += chunk.toString('utf8')
+      let nl
+      while ((nl = buffer.indexOf('\n')) !== -1) {
+        const line = buffer.slice(0, nl)
+        buffer = buffer.slice(nl + 1)
+        if (!parseLine(line, (row) => this.push(row), callback)) return
+      }
+      callback()
+    },
+    flush(callback) {
+      if (parseLine(buffer, (row) => this.push(row), callback)) callback()
+    },
+  })
+}
+
+/**
+ * Build a `RowReadable` from a connection's transport client.
+ * @param {object} client transport exposing `streamSelect`.
+ * @param {string} sql read-only SELECT to stream.
+ * @param {{ signal?: AbortSignal, cursor?: string }} [opts]
+ * @returns {RowReadable}
+ */
+export function createSelectStream(client, sql, opts = {}) {
+  if (typeof client.streamSelect !== 'function') {
+    throw new RedDBError(
+      'STREAMING_UNSUPPORTED',
+      'the active transport does not support streaming reads (use red:// or http(s)://)',
+    )
+  }
+  if (opts.cursor == null && (typeof sql !== 'string' || sql.trim().length === 0)) {
+    throw new RedDBError('INVALID_STREAM_QUERY', 'stream() requires a non-empty SQL string')
+  }
+  const sessionPromise = Promise.resolve().then(() =>
+    client.streamSelect({ sql, cursor: opts.cursor, signal: opts.signal }),
+  )
+  return new RowReadable(sessionPromise, { signal: opts.signal })
+}
+
+/**
+ * Build a `RowWritable` from a connection's transport client.
+ * @param {object} client transport exposing `streamInput`.
+ * @param {string} target table to ingest into.
+ * @param {{ signal?: AbortSignal, columns?: string[] }} [opts] `columns`
+ *   fixes the ingest column set; when omitted it is inferred from the
+ *   keys of the first row written.
+ * @returns {RowWritable}
+ */
+export function createInputStream(client, target, opts = {}) {
+  if (typeof client.streamInput !== 'function') {
+    throw new RedDBError(
+      'STREAMING_UNSUPPORTED',
+      'the active transport does not support streaming writes (use red:// or http(s)://)',
+    )
+  }
+  if (typeof target !== 'string' || target.trim().length === 0) {
+    throw new RedDBError('INVALID_STREAM_TARGET', 'inputStream() requires a non-empty target table')
+  }
+  const sessionPromise = Promise.resolve().then(() =>
+    client.streamInput({
+      target,
+      columns: opts.columns,
+      signal: opts.signal,
+    }),
+  )
+  return new RowWritable(sessionPromise, { signal: opts.signal })
+}

package/src/url.js

@@ -1,271 +1,12 @@
 /**
- * `red://` connection-string parser.
- *
- * One URL covers every transport RedDB speaks:
- *
- *   red://                                          embedded in-memory
- *   red:///abs/path/data.rdb                        embedded persistent
- *   red://user:pass@host:5050                       remote, default proto=red (wire)
- *   red://host:8080?proto=https                     remote HTTPS
- *   red://host:5432?proto=pg                        PostgreSQL wire
- *   red://host:5055?proto=grpc&token=sk-abc         remote gRPC w/ bearer
- *   red://host:8080?proto=https&apiKey=ak-xyz       remote HTTPS w/ api key
- *
- * Backwards-compat: legacy `memory://`, `file://`, `grpc://` URLs
- * still work via `parseLegacyUrl`. New code should prefer `red://`
- * because it carries auth + protocol selection in one place.
+ * Compatibility shim. The `red://` connection-string parser now lives in
+ * the transport-agnostic core (`core/url.js`); this file keeps the
+ * historical `./url.js` import path working.
  */
 
-import { RedDBError } from './protocol.js'
-
-/**
- * @typedef {object} ParsedUri
- * @property {'embedded' | 'http' | 'https' | 'red' | 'reds' | 'grpc' | 'grpcs' | 'pg'} kind
- * @property {string} [host]
- * @property {number} [port]
- * @property {string} [path]            // for embedded `file://`-equivalent
- * @property {string} [username]
- * @property {string} [password]
- * @property {string} [token]
- * @property {string} [apiKey]
- * @property {string} [loginUrl]        // explicit override for login flow
- * @property {URLSearchParams} [params] // remaining query params
- * @property {string} originalUri
- */
-
-/**
- * Parse any URI string into a normalised `ParsedUri`.
- * Accepts `red://`, `memory://`, `file://`, `grpc://` (the latter
- * three for backwards compat).
- *
- * @param {string} uri
- * @returns {ParsedUri}
- */
-export function parseUri(uri) {
-  if (typeof uri !== 'string' || uri.length === 0) {
-    throw new TypeError(
-      "connect() requires a URI string (e.g. 'red://localhost:5050' or 'red:///data.rdb')",
-    )
-  }
-  if (uri.startsWith('red://') || uri === 'red:' || uri === 'red:/') {
-    return parseRedUrl(uri)
-  }
-  return parseLegacyUrl(uri)
-}
-
-/**
- * Parse a `red://` URL.
- *
- * Authority shape: `[user[:pass]@]host[:port]`
- * Path: optional, used as filesystem path when `host` is absent or
- * is the special token `localhost-embedded` (rare).
- * Query: `proto`, `token`, `apiKey`, `loginUrl`.
- */
-export function parseRedUrl(uri) {
-  // The host might be missing (`red:///path`), the URL constructor
-  // requires *something* there. Re-write to a parse-friendly shape:
-  // - `red:///x`     → `red://embedded.local/x`   (embedded with path)
-  // - `red://memory` → `red://embedded.local`     (embedded in-memory)
-  // - `red://`       → `red://embedded.local`     (embedded in-memory)
-  let normalised = uri
-  if (uri === 'red:' || uri === 'red:/' || uri === 'red://') {
-    normalised = 'red://embedded.local'
-  } else if (uri.startsWith('red:///')) {
-    normalised = `red://embedded.local${uri.slice('red://'.length)}`
-  } else if (
-    uri === 'red://memory'
-    || uri === 'red://memory/'
-    || uri === 'red://:memory'
-    || uri === 'red://:memory:'  // SQLite-style ":memory:" alias
-  ) {
-    normalised = 'red://embedded.local'
-  }
-
-  let parsed
-  try {
-    parsed = new URL(normalised)
-  } catch (err) {
-    throw new RedDBError('UNPARSEABLE_URI', `failed to parse '${uri}': ${err.message}`)
-  }
-
-  const params = parsed.searchParams
-  const proto = (params.get('proto') || '').toLowerCase()
-  const path = parsed.pathname && parsed.pathname !== '/' ? parsed.pathname : ''
-
-  // Embedded: special host, OR `proto=embedded`, OR no proto + has path
-  // and the user clearly meant a file path (red:///abs/path).
-  if (parsed.hostname === 'embedded.local') {
-    if (path) {
-      return {
-        kind: 'embedded',
-        path,
-        params,
-        originalUri: uri,
-      }
-    }
-    return {
-      kind: 'embedded',
-      params,
-      originalUri: uri,
-    }
-  }
-
-  // Remote — default proto is red (wire).
-  const kind = resolveKind(proto)
-  const port = parsed.port ? Number(parsed.port) : defaultPortFor(kind)
-  const username = parsed.username ? decodeURIComponent(parsed.username) : undefined
-  const password = parsed.password ? decodeURIComponent(parsed.password) : undefined
-
-  return {
-    kind,
-    host: parsed.hostname,
-    port,
-    path: path || undefined,
-    username,
-    password,
-    token: params.get('token') ?? undefined,
-    apiKey: params.get('apiKey') ?? params.get('api_key') ?? undefined,
-    loginUrl: params.get('loginUrl') ?? params.get('login_url') ?? undefined,
-    params,
-    originalUri: uri,
-  }
-}
-
-/**
- * Backwards-compat parser for the legacy URL shapes the driver
- * accepted before `red://` existed. Returns the same `ParsedUri`
- * shape so downstream code is uniform.
- */
-export function parseLegacyUrl(uri) {
-  if (uri === 'memory://' || uri === 'memory:') {
-    return { kind: 'embedded', originalUri: uri }
-  }
-  if (uri.startsWith('file://')) {
-    const path = uri.slice('file://'.length)
-    if (!path) {
-      throw new TypeError(`invalid file:// URI: missing path in '${uri}'`)
-    }
-    return { kind: 'embedded', path, originalUri: uri }
-  }
-  if (
-    uri.startsWith('grpc://')
-    || uri.startsWith('grpcs://')
-    || uri.startsWith('reds://')
-  ) {
-    const scheme = uri.split('://', 1)[0]
-    const stripped = uri.slice(`${scheme}://`.length)
-    const [hostPort] = stripped.split(/[/?]/, 1)
-    const [host, portStr] = hostPort.split(':')
-    if (!host) {
-      throw new TypeError(`invalid ${scheme}:// URI: missing host in '${uri}'`)
-    }
-    const legacyKind = scheme === 'reds' ? 'reds' : scheme === 'grpcs' ? 'grpcs' : scheme === 'grpc' ? 'grpc' : 'red'
-    return {
-      kind: legacyKind,
-      host,
-      port: portStr ? Number(portStr) : defaultPortFor(legacyKind),
-      originalUri: uri,
-    }
-  }
-  if (uri.startsWith('http://') || uri.startsWith('https://')) {
-    let parsed
-    try {
-      parsed = new URL(uri)
-    } catch (err) {
-      throw new RedDBError('UNPARSEABLE_URI', `failed to parse '${uri}': ${err.message}`)
-    }
-    return {
-      kind: parsed.protocol === 'https:' ? 'https' : 'http',
-      host: parsed.hostname,
-      port: parsed.port ? Number(parsed.port) : defaultPortFor(parsed.protocol === 'https:' ? 'https' : 'http'),
-      path: parsed.pathname !== '/' ? parsed.pathname : undefined,
-      username: parsed.username ? decodeURIComponent(parsed.username) : undefined,
-      password: parsed.password ? decodeURIComponent(parsed.password) : undefined,
-      token: parsed.searchParams.get('token') ?? undefined,
-      apiKey: parsed.searchParams.get('apiKey') ?? undefined,
-      params: parsed.searchParams,
-      originalUri: uri,
-    }
-  }
-  throw new RedDBError(
-    'UNSUPPORTED_SCHEME',
-    `unsupported URI: '${uri}'. Use 'red://...' or one of memory://, file://, grpc://, http(s)://`,
-  )
-}
-
-function resolveKind(protoQueryParam) {
-  switch (protoQueryParam) {
-    case '':
-    case 'red':
-      return 'red'
-    case 'reds':
-      return 'reds'
-    case 'grpc':
-      return 'grpc'
-    case 'grpcs':
-      return 'grpcs'
-    case 'http':
-      return 'http'
-    case 'https':
-      return 'https'
-    case 'pg':
-    case 'postgres':
-    case 'postgresql':
-      return 'pg'
-    default:
-      throw new RedDBError(
-        'UNSUPPORTED_PROTO',
-        `unknown proto='${protoQueryParam}'. Supported: red | reds | grpc | grpcs | http | https | pg`,
-      )
-  }
-}
-
-function defaultPortFor(kind) {
-  switch (kind) {
-    case 'http':
-      return 8080
-    case 'https':
-      return 8443
-    case 'red':
-    case 'reds':
-    case 'redwire':
-      return 5050
-    case 'grpc':
-      return 5055
-    case 'grpcs':
-      return 5056
-    case 'pg':
-    case 'postgres':
-    case 'postgresql':
-      return 5432
-    default:
-      return undefined
-  }
-}
-
-/**
- * Derive the HTTP login URL (`/auth/login`) from a parsed URI.
- * Used by the auto-login flow when the user supplies `username:password@`
- * but not an explicit `loginUrl`.
- *
- * Strategy: if proto is already http/https, just append `/auth/login`.
- * For grpc/grpcs/pg, default to https://host:443 — operators that
- * don't want that should pass `loginUrl=` explicitly.
- */
-export function deriveLoginUrl(parsed) {
-  if (parsed.loginUrl) return parsed.loginUrl
-  if (!parsed.host) {
-    throw new RedDBError(
-      'AUTH_LOGIN_NEEDS_HOST',
-      'cannot derive loginUrl without a host; pass it explicitly via loginUrl=...',
-    )
-  }
-  if (parsed.kind === 'http' || parsed.kind === 'https') {
-    const scheme = parsed.kind
-    const port = parsed.port ?? defaultPortFor(parsed.kind)
-    return `${scheme}://${parsed.host}:${port}/auth/login`
-  }
-  // Non-HTTP transports — default to HTTPS on 443 unless the user
-  // tells us otherwise.
-  return `https://${parsed.host}/auth/login`
-}
+export {
+  parseUri,
+  parseRedUrl,
+  parseLegacyUrl,
+  deriveLoginUrl,
+} from './core/url.js'