@lossless.org/client 0.1.0

package/ts/objectstorage/internal.multipart.types.ts

@@ -0,0 +1,106 @@
+export interface IMultipartPlanningOptions {
+  objectSize: number;
+  requestedPartSize?: number;
+  requestedConcurrency?: number;
+}
+
+interface IMultipartUploadPlanBase {
+  objectSize: number;
+  concurrency: number;
+  maximumBufferedBytes: number;
+}
+
+export interface IMultipartZeroUploadPlan extends IMultipartUploadPlanBase {
+  classification: 'zero';
+  objectSize: 0;
+  partSize: 0;
+  partCount: 0;
+  maximumBufferedBytes: 0;
+}
+
+export interface IMultipartDataUploadPlan extends IMultipartUploadPlanBase {
+  classification: 'multipart';
+  partSize: number;
+  partCount: number;
+}
+
+export type TMultipartUploadPlan = IMultipartZeroUploadPlan | IMultipartDataUploadPlan;
+
+export interface IMultipartStreamPart {
+  partNumber: number;
+  byteOffset: number;
+  isLastPart: boolean;
+  /**
+   * An exact, exclusively owned copy. The consumer never mutates or reuses it.
+   */
+  data: Uint8Array;
+  signal: AbortSignal;
+}
+
+export interface IConsumeExactMultipartStreamOptions {
+  readableStream: ReadableStream<Uint8Array>;
+  plan: TMultipartUploadPlan;
+  onPart: (part: IMultipartStreamPart) => Promise<void>;
+  signal?: AbortSignal;
+  /** Internal failure-teardown bound (default and maximum: 30000ms). */
+  failureSettlementTimeoutMs?: number;
+}
+
+export interface IMultipartStreamConsumptionResult {
+  completionAuthorized: true;
+  bytesConsumed: number;
+  partsConsumed: number;
+  peakBufferedBytes: number;
+  maximumBufferedBytes: number;
+}
+
+export type TMultipartStreamFailureKind =
+  | 'aborted'
+  | 'source-read'
+  | 'source-buffer-limit'
+  | 'source-empty-chunk-limit'
+  | 'source-short'
+  | 'source-extra'
+  | 'part-callback';
+
+export type TMultipartOwnedReaderCleanup =
+  | {
+      status: 'closed';
+    }
+  | {
+      status: 'cancelled';
+    }
+  | {
+      status: 'cancel-failed';
+      cause: unknown;
+    }
+  | {
+      status: 'cancel-detached';
+    };
+
+export interface IMultipartOwnedCleanupState {
+  callbacks: 'settled' | 'detached';
+  reader: TMultipartOwnedReaderCleanup;
+}
+
+export type TMultipartRetryState =
+  | {
+      disposition: 'safe-to-retry';
+      requiresMultipartAbort: false;
+    }
+  | {
+      disposition: 'abort-multipart-first';
+      requiresMultipartAbort: true;
+    };
+
+export interface IMultipartStreamFailureState {
+  completionAuthorized: false;
+  kind: TMultipartStreamFailureKind;
+  cause?: unknown;
+  expectedBytes: number;
+  observedBytes: number;
+  partCallbacksStarted: number;
+  completedPartNumbers: number[];
+  cleanup: IMultipartOwnedCleanupState;
+  retry: TMultipartRetryState;
+}

package/ts/objectstorage/plugins.ts

@@ -0,0 +1,35 @@
+// plugins.ts
+
+// node native
+import * as crypto from 'node:crypto';
+import * as path from 'node:path';
+import * as stream from 'node:stream';
+
+export { crypto, path, stream };
+
+// @push.rocks scope
+import * as smartmime from '@push.rocks/smartmime';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
+import * as smartstream from '@push.rocks/smartstream';
+import * as smartstring from '@push.rocks/smartstring';
+import * as smartunique from '@push.rocks/smartunique';
+
+export { smartmime, smartpath, smartpromise, smartrx, smartstream, smartstring, smartunique };
+
+// @tsclass
+import * as tsclass from '@tsclass/tsclass';
+
+export {
+  tsclass,
+}
+
+// third party scope
+import * as s3 from '@aws-sdk/client-s3';
+import { Minimatch } from 'minimatch';
+
+export {
+  s3,
+  Minimatch,
+}

package/ts/plugins.ts

@@ -0,0 +1,4 @@
+export const loadNoSql = () => import('./nosqldb/classes.connection.js');
+export const loadObjects = () => import('./objectstorage/classes.connection.js');
+export const loadSql = () => import('./sqldb/classes.sqlconnection.js');
+export const loadClickHouse = () => import('./sqldb/classes.clickhouseconnection.js');

package/ts/sqldb/classes.clickhouseconnection.ts

@@ -0,0 +1,189 @@
+import * as plugins from './plugins.clickhouse.js';
+import { Operation } from '../core/classes.operation.js';
+import { LosslessClientError, backendError } from '../core/classes.error.js';
+import type { ICapabilities, IOperationOptions, IReadiness } from '../core/interfaces.js';
+import { quoteClickHouseIdentifier as quoteIdentifier, resultLimits, valueBytes } from './interfaces.js';
+import type { IClickHouseConnectionOptions, IClickHouseStatement, IInsertOptions, IQueryOptions } from './interfaces.js';
+import { SmartClickHouseDb } from './clickhouse/smartclickhouse.classes.smartclickhouse.js';
+
+/** ClickHouse HTTP connection. The server executes filtering, projection and aggregation. */
+export class ClickHouseConnection {
+  public readonly backend = 'clickhouse' as const;
+  public readonly metrics: SmartClickHouseDb;
+  public readonly capabilities: ICapabilities = Object.freeze({ backend: 'clickhouse',
+    transactions: 'unsupported', changeStreams: 'unsupported', objectMetrics: 'unsupported', diskResidentScans: 'available' });
+  private client?: plugins.clickhouse.ClickHouseClient;
+  private readonly operations = new Map<Operation, () => void>();
+  private readonly tasks = new Set<Promise<unknown>>();
+  private closed = false;
+  private closing?: Promise<void>;
+  private connecting?: Promise<void>;
+  private readonly writerCleanup = new plugins.asyncHooks.AsyncLocalStorage<boolean>();
+
+  constructor(public readonly options: IClickHouseConnectionOptions) {
+    const url = new URL(options.url);
+    if (!['http:', 'https:'].includes(url.protocol) || !options.database || url.username || url.password ||
+      url.search || url.hash || url.pathname !== '/') {
+      throw new LosslessClientError('invalid_argument', 'ClickHouse requires an HTTP(S) URL, separate credentials and a database.');
+    }
+    const limit = options.connectionLimit ?? 10;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1024) {
+      throw new LosslessClientError('invalid_argument', 'connectionLimit must be between 1 and 1024.');
+    }
+    const budget = new Operation({ timeoutMs: options.timeoutMs }); budget.dispose();
+    this.metrics = new SmartClickHouseDb(options, this);
+  }
+
+  public connect(options: IOperationOptions = {}): Promise<void> {
+    if (this.closed || this.closing) return Promise.reject(new LosslessClientError('closed', 'ClickHouse connection is closed.'));
+    if (this.connecting) return this.connecting;
+    if (this.client) return Promise.resolve();
+    this.client = plugins.clickhouse.createClient({ url: this.options.url,
+      database: this.options.database, username: this.options.username, password: this.options.password,
+      log: { level: plugins.clickhouse.ClickHouseLogLevel.OFF },
+      max_open_connections: this.options.connectionLimit ?? 10, request_timeout: this.options.timeoutMs ?? 30_000 });
+    this.connecting = (async () => {
+      try { await this.query({ sql: 'SELECT 1 AS ready' }, { ...options, maxRows: 1 }); }
+      catch (error) { const client = this.client; this.client = undefined; await client?.close(); throw error; }
+      finally { this.connecting = undefined; }
+    })();
+    return this.connecting;
+  }
+
+  private begin(options: IOperationOptions): { client: plugins.clickhouse.ClickHouseClient; operation: Operation } {
+    if (!this.client || this.closed || (this.closing && !this.writerCleanup.getStore())) {
+      throw new LosslessClientError(this.closed || this.closing ? 'closed' : 'not_connected', 'ClickHouse connection is not open.');
+    }
+    const operation = new Operation(options, this.options.timeoutMs);
+    this.operations.set(operation, () => {});
+    try { operation.check(); }
+    catch (error) { this.end(operation); throw error; }
+    return { client: this.client, operation };
+  }
+
+  private end(operation: Operation): void {
+    operation.dispose();
+    this.operations.delete(operation);
+  }
+
+  private track<Result>(task: Promise<Result>): Promise<Result> {
+    this.tasks.add(task);
+    void task.finally(() => this.tasks.delete(task)).catch(() => {});
+    return task;
+  }
+
+  public async ready(options: IOperationOptions = {}): Promise<IReadiness> {
+    try { await this.query({ sql: 'SELECT 1 AS ready' }, { ...options, maxRows: 1 }); return { ready: true }; }
+    catch (error) { return { ready: false, reason: error instanceof LosslessClientError ? error.code : 'unavailable' }; }
+  }
+
+  public execute(statement: IClickHouseStatement, options: IOperationOptions = {}): Promise<{ queryId: string; completion: 'acknowledged' }> {
+    this.validate(statement);
+    return this.track((async () => {
+      const { client, operation } = this.begin(options);
+      try {
+        const result = await client.command({ query: statement.sql, query_params: statement.values,
+          abort_signal: operation.signal, clickhouse_settings: { wait_end_of_query: 1, mutations_sync: '2' } });
+        return { queryId: result.query_id, completion: 'acknowledged' as const };
+      } catch (error) {
+        // HTTP transport loss does not establish whether the server committed a mutation.
+        if (operation.signal.aborted || !(error instanceof Error && 'code' in error && /^\d+$/.test(String(error.code)))) {
+          throw new LosslessClientError('ambiguous_write', 'ClickHouse mutation outcome is unknown.', { cause: error });
+        }
+        throw backendError(error);
+      } finally { this.end(operation); }
+    })());
+  }
+
+  public async *stream<Row = Record<string, unknown>>(statement: IClickHouseStatement,
+    options: IOperationOptions = {}): AsyncGenerator<Row> {
+    this.validate(statement);
+    const { client, operation } = this.begin(options);
+    let result: Awaited<ReturnType<typeof client.query<'JSONEachRow'>>> | undefined;
+    try {
+      result = await this.track(client.query({ query: statement.sql, query_params: statement.values,
+        format: 'JSONEachRow', abort_signal: operation.signal,
+        clickhouse_settings: { output_format_json_quote_64bit_integers: 1, output_format_json_quote_decimals: 1 } }));
+      const current = result;
+      this.operations.set(operation, () => current.close());
+      const stream = result.stream<Row>();
+      stream.on('error', () => {});
+      stream.once('close', () => this.end(operation));
+      for await (const batch of stream) {
+        operation.check();
+        for (const row of batch) { operation.check(); yield row.json(); }
+      }
+      operation.check();
+    } catch (error) { throw operation.signal.aborted ? operation.signal.reason : backendError(error); }
+    finally { result?.close(); this.end(operation); }
+  }
+
+  public async query<Row = Record<string, unknown>>(statement: IClickHouseStatement,
+    options: IQueryOptions = {}): Promise<Row[]> {
+    const limits = resultLimits(options), rows: Row[] = [];
+    let bytes = 0;
+    for await (const row of this.stream<Row>(statement, options)) {
+      bytes += valueBytes(row);
+      if (rows.length >= limits.rows || bytes > limits.bytes) {
+        throw new LosslessClientError('result_limit', 'ClickHouse result exceeded its materialization limit. Use stream().');
+      }
+      rows.push(row);
+    }
+    return rows;
+  }
+
+  public insert(table: string, rows: AsyncIterable<Record<string, unknown>> | Iterable<Record<string, unknown>>,
+    options: IInsertOptions = {}): Promise<{ queryId: string; completion: 'acknowledged' }> {
+    return this.insertQualified(this.options.database, table, rows, options);
+  }
+
+  public insertQualified(database: string, table: string,
+    rows: AsyncIterable<Record<string, unknown>> | Iterable<Record<string, unknown>>,
+    options: IInsertOptions = {}): Promise<{ queryId: string; completion: 'acknowledged' }> {
+    const target = `${quoteIdentifier(database)}.${quoteIdentifier(table)}`;
+    const maxBytes = options.maxBatchBytes ?? 4 * 1024 * 1024;
+    if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) throw new LosslessClientError('invalid_argument', 'maxBatchBytes must be a positive integer.');
+    return this.track((async () => {
+      const { client, operation } = this.begin(options);
+      const source = plugins.stream.Readable.from((async function* () {
+        for await (const row of operation.iterate(rows)) {
+          operation.check();
+          if (valueBytes(row) > maxBytes) throw new LosslessClientError('invalid_argument', 'ClickHouse row exceeds maxBatchBytes.');
+          yield row;
+        }
+      })(), { objectMode: true, highWaterMark: 1 });
+      source.on('error', () => {});
+      this.operations.set(operation, () => source.destroy());
+      try {
+        const result = await client.insert({ table: target, values: source, format: 'JSONEachRow',
+          abort_signal: operation.signal, clickhouse_settings: { async_insert: 0, wait_end_of_query: 1 } });
+        return { queryId: result.query_id, completion: 'acknowledged' as const };
+      } catch (error) {
+        throw new LosslessClientError('ambiguous_write', 'ClickHouse insert may have stored part or all of the input.', { cause: error });
+      } finally { source.destroy(); this.end(operation); }
+    })());
+  }
+
+  public close(): Promise<void> {
+    return this.closing ??= (async () => {
+      let writerError: unknown;
+      try { await this.writerCleanup.run(true, () => this.metrics.closeWriters()); } catch (error) { writerError = error; }
+      this.closed = true;
+      for (const [operation, dispose] of this.operations) { operation.cancel(); dispose(); }
+      await Promise.allSettled([...this.tasks]);
+      if (this.connecting) await Promise.allSettled([this.connecting]);
+      const client = this.client;
+      this.client = undefined;
+      await client?.close();
+      for (const operation of this.operations.keys()) this.end(operation);
+      if (writerError) throw writerError;
+    })();
+  }
+
+  private validate(statement: IClickHouseStatement): void {
+    if (!statement || typeof statement.sql !== 'string' || !statement.sql.trim() ||
+        (statement.values !== undefined && (!statement.values || typeof statement.values !== 'object' || Array.isArray(statement.values)))) {
+      throw new LosslessClientError('invalid_argument', 'ClickHouse requires SQL text and a named parameter object.');
+    }
+  }
+}