@lossless.org/client 0.1.0

package/ts/sqldb/classes.sqlconnection.ts

@@ -0,0 +1,350 @@
+import * as plugins from './plugins.mariadb.js';
+import { SqlTable } from './classes.sqltable.js';
+import { SqlPool, type SqlPooledConnection } from './internal.sqlpool.js';
+import { Operation } from '../core/classes.operation.js';
+import { LosslessClientError, backendError } from '../core/classes.error.js';
+import type { ICapabilities, IOperationOptions, IReadiness } from '../core/interfaces.js';
+import { quoteIdentifier, resultLimits, valueBytes } from './interfaces.js';
+import type { IInsertOptions, IQueryOptions, ISqlConnectionOptions, ISqlMutationResult, ISqlStatement, TSqlValue } from './interfaces.js';
+
+interface ILease {
+  connection: SqlPooledConnection;
+  operation: Operation;
+  close: () => Promise<void>;
+  done: Promise<void>;
+}
+
+const validateStatement = (statement: ISqlStatement): void => {
+  if (!statement || typeof statement.sql !== 'string' || !statement.sql.trim()) {
+    throw new LosslessClientError('invalid_argument', 'A SQL statement is required.');
+  }
+  if (statement.values !== undefined && !Array.isArray(statement.values)) {
+    throw new LosslessClientError('invalid_argument', 'SQL values must be an array.');
+  }
+  for (const value of statement.values ?? []) {
+    if (value !== null && !['string', 'number', 'bigint', 'boolean'].includes(typeof value) &&
+      !(value instanceof Date) && !(value instanceof Uint8Array)) {
+      throw new LosslessClientError('invalid_argument', 'SQL parameters must be supported scalar, date or binary values.');
+    }
+    if (value instanceof Date && !Number.isFinite(value.getTime())) {
+      throw new LosslessClientError('invalid_argument', 'Invalid date parameter.');
+    }
+    if (typeof value === 'number' && (!Number.isFinite(value) ||
+        (Number.isInteger(value) && !Number.isSafeInteger(value)))) {
+      throw new LosslessClientError('invalid_argument', 'Use bigint or a decimal string for exact large numbers.');
+    }
+  }
+};
+
+const boundValues = (statement: ISqlStatement): TSqlValue[] =>
+  (statement.values ?? []).map(value => value instanceof Date
+    ? value.toISOString().replace('T', ' ').replace('Z', '')
+    : value instanceof Uint8Array ? Buffer.from(value) : value);
+
+const mutationResult = (result: plugins.mariadb.UpsertResult): ISqlMutationResult => ({
+  affectedRows: result.affectedRows,
+  ...(result.insertId === undefined ? {} : { insertId: BigInt(result.insertId) }),
+  completion: 'acknowledged',
+});
+
+/** One MariaDB-protocol pool. SQLDB uses the same wire adapter with its own capabilities. */
+export class SqlConnection {
+  public readonly backend: 'sqldb' | 'mariadb';
+  public readonly capabilities: ICapabilities;
+  private pool?: SqlPool;
+  private closing?: Promise<void>;
+  private connecting?: Promise<void>;
+  private closed = false;
+  private readonly acquisitions = new Set<Promise<ILease>>();
+  private readonly leases = new Set<ILease>();
+  private readonly pendingOperations = new Set<Operation>();
+
+  constructor(public readonly options: ISqlConnectionOptions) {
+    if (!['sqldb', 'mariadb'].includes(options.backend) || !options.host || !options.database || !options.user) {
+      throw new LosslessClientError('invalid_argument', 'SQL backend, host, user and database are required.');
+    }
+    if (options.backend === 'sqldb' && options.tls) {
+      throw new LosslessClientError('unsupported_capability', 'SQLDB does not yet support TLS.');
+    }
+    this.backend = options.backend;
+    this.capabilities = Object.freeze({ backend: this.backend, transactions: 'available',
+      changeStreams: 'unsupported', objectMetrics: 'unsupported',
+      diskResidentScans: this.backend === 'sqldb' ? 'unsupported' : 'available' });
+  }
+
+  public connect(options: IOperationOptions = {}): Promise<void> {
+    if (this.closed || this.closing) return Promise.reject(new LosslessClientError('closed', 'SQL connection is closed.'));
+    if (this.connecting) return this.connecting;
+    if (this.pool) return Promise.resolve();
+    const limit = this.options.connectionLimit ?? 10;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1024) {
+      return Promise.reject(new LosslessClientError('invalid_argument', 'connectionLimit must be between 1 and 1024.'));
+    }
+    this.pool = new SqlPool({
+      host: this.options.host, port: this.options.port ?? 3306, user: this.options.user,
+      password: this.options.password, database: this.options.database,
+      connectTimeout: this.options.timeoutMs ?? 30_000,
+      ssl: this.options.tls, bigIntAsNumber: false, decimalAsNumber: false,
+      dateStrings: true, timezone: this.backend === 'mariadb' ? '+00:00' : undefined,
+      multipleStatements: false,
+    }, limit);
+    this.connecting = (async () => {
+      try { await this.query({ sql: 'SELECT 1 AS ready' }, { ...options, maxRows: 1 }); }
+      catch (error) {
+        const pool = this.pool;
+        this.pool = undefined;
+        await pool?.end();
+        throw error;
+      } finally { this.connecting = undefined; }
+    })();
+    return this.connecting;
+  }
+
+  public async ready(options: IOperationOptions = {}): Promise<IReadiness> {
+    try { await this.query({ sql: 'SELECT 1 AS ready' }, { ...options, maxRows: 1 }); return { ready: true }; }
+    catch (error) { return { ready: false, reason: error instanceof LosslessClientError ? error.code : 'unavailable' }; }
+  }
+
+  public table<Row extends { [Key in keyof Row]: TSqlValue }>(name: string): SqlTable<Row> {
+    return new SqlTable<Row>(this, name);
+  }
+
+  private acquire(options: IOperationOptions): Promise<ILease> {
+    if (!this.pool || this.closed || this.closing) {
+      return Promise.reject(new LosslessClientError(this.closed ? 'closed' : 'not_connected', 'SQL connection is not open.'));
+    }
+    const pool = this.pool;
+    const operation = new Operation(options, this.options.timeoutMs);
+    this.pendingOperations.add(operation);
+    const acquisition = (async (): Promise<ILease> => {
+      let connection: SqlPooledConnection | undefined;
+      try {
+        operation.check();
+        connection = await pool.getConnection(operation);
+        operation.check();
+        const acquired = connection;
+        const settled = Promise.withResolvers<void>();
+        let disposal: Promise<void> | undefined;
+        const abort = () => acquired.destroy();
+        const lease: ILease = { connection: acquired, operation, done: settled.promise,
+          close: () => disposal ??= (async () => {
+            try { await acquired.release(); }
+            finally {
+              operation.signal.removeEventListener('abort', abort);
+              operation.dispose();
+              this.pendingOperations.delete(operation);
+              this.leases.delete(lease);
+              settled.resolve();
+            }
+          })(),
+        };
+        operation.signal.addEventListener('abort', abort, { once: true });
+        this.leases.add(lease);
+        return lease;
+      } catch (error) {
+        connection?.destroy();
+        await connection?.release();
+        operation.dispose();
+        this.pendingOperations.delete(operation);
+        throw operation.signal.aborted ? operation.signal.reason : backendError(error);
+      }
+    })();
+    this.acquisitions.add(acquisition);
+    void acquisition.finally(() => this.acquisitions.delete(acquisition)).catch(() => {});
+    return acquisition;
+  }
+
+  public async execute(statement: ISqlStatement, options: IOperationOptions = {}): Promise<ISqlMutationResult> {
+    validateStatement(statement);
+    const lease = await this.acquire(options);
+    let dispatched = false;
+    try {
+      lease.operation.check();
+      dispatched = true;
+      const result = await lease.connection.execute<plugins.mariadb.UpsertResult>(statement.sql, boundValues(statement));
+      return mutationResult(result);
+    } catch (error) {
+      if (dispatched && (lease.operation.signal.aborted || !lease.connection.isValid())) {
+        throw new LosslessClientError('ambiguous_write', 'SQL write outcome is unknown after connection loss.', { cause: error });
+      }
+      throw backendError(error);
+    } finally { await lease.close(); }
+  }
+
+  public async query<Row = Record<string, unknown>>(statement: ISqlStatement, options: IQueryOptions = {}): Promise<Row[]> {
+    const limits = resultLimits(options);
+    const rows: Row[] = [];
+    let bytes = 0;
+    for await (const row of this.stream<Row>(statement, options)) {
+      bytes += valueBytes(row);
+      if (rows.length >= limits.rows || bytes > limits.bytes) {
+        throw new LosslessClientError('result_limit', 'SQL result exceeded its materialization limit. Use stream().');
+      }
+      rows.push(row);
+    }
+    return rows;
+  }
+
+  public async *stream<Row = Record<string, unknown>>(statement: ISqlStatement,
+    options: IOperationOptions = {}): AsyncGenerator<Row> {
+    validateStatement(statement);
+    const lease = await this.acquire(options);
+    let readable: plugins.stream.Readable | undefined;
+    let completed = false;
+    try {
+      lease.operation.check();
+      readable = lease.connection.queryStream(statement.sql, boundValues(statement));
+      // A paused consumer still owns a socket. Stream close independently returns its lease.
+      readable.on('error', () => {});
+      readable.once('close', () => { void lease.close().catch(() => {}); });
+      for await (const row of readable) {
+        lease.operation.check();
+        yield row as Row;
+      }
+      lease.operation.check();
+      completed = true;
+    } catch (error) {
+      throw lease.operation.signal.aborted ? lease.operation.signal.reason : backendError(error);
+    } finally {
+      if (!completed) lease.connection.destroy();
+      readable?.destroy();
+      await lease.close();
+    }
+  }
+
+  public async insert(table: string, rows: AsyncIterable<Record<string, TSqlValue>> | Iterable<Record<string, TSqlValue>>,
+    options: IInsertOptions = {}): Promise<ISqlMutationResult> {
+    const tableName = quoteIdentifier(table);
+    const batchSize = options.batchSize ?? 500;
+    const maxBytes = options.maxBatchBytes ?? 4 * 1024 * 1024;
+    if (!Number.isSafeInteger(batchSize) || batchSize < 1 || !Number.isSafeInteger(maxBytes) || maxBytes < 1) {
+      throw new LosslessClientError('invalid_argument', 'Insert batch limits must be positive integers.');
+    }
+    const operation = new Operation(options, this.options.timeoutMs);
+    if (!this.pool || this.closed || this.closing) {
+      operation.dispose();
+      throw new LosslessClientError(this.closed ? 'closed' : 'not_connected', 'SQL connection is not open.');
+    }
+    this.pendingOperations.add(operation);
+    let columns: string[] | undefined;
+    let batch: TSqlValue[][] = [], bytes = 0, affectedRows = 0;
+    const flush = async () => {
+      if (!batch.length) return;
+      const result = await this.execute({ sql: `INSERT INTO ${tableName} (${columns!.map(quoteIdentifier).join(',')}) VALUES ${batch.map(() => '(' + columns!.map(() => '?').join(',') + ')').join(',')}`,
+        values: batch.flat() }, { signal: operation.signal, timeoutMs: options.timeoutMs });
+      affectedRows += result.affectedRows;
+      batch = []; bytes = 0;
+    };
+    try {
+      for await (const row of operation.iterate(rows)) {
+        operation.check();
+        columns ??= Object.keys(row);
+        if (!columns.length || Object.keys(row).length !== columns.length || columns.some(key => !Object.hasOwn(row, key))) {
+          throw new LosslessClientError('invalid_argument', 'Every inserted row must have the same columns.');
+        }
+        const size = valueBytes(row);
+        if (size > maxBytes) throw new LosslessClientError('invalid_argument', 'Insert row exceeds maxBatchBytes.');
+        if (batch.length && (batch.length >= batchSize || bytes + size > maxBytes)) await flush();
+        batch.push(columns.map(key => row[key])); bytes += size;
+      }
+      await flush();
+      return { affectedRows, completion: 'acknowledged' };
+    } catch (error) {
+      if (affectedRows) throw new LosslessClientError(error instanceof LosslessClientError && error.code === 'ambiguous_write'
+        ? 'ambiguous_write' : 'partial_write', 'Bulk insert stopped after acknowledged batches.', { cause: error, acknowledgedRows: affectedRows });
+      throw error;
+    } finally { operation.dispose(); this.pendingOperations.delete(operation); }
+  }
+
+  public async transaction<Result>(callback: (transaction: SqlTransaction) => Promise<Result>,
+    options: IOperationOptions = {}): Promise<Result> {
+    const lease = await this.acquire(options);
+    const transaction = new SqlTransaction(lease);
+    let committing = false;
+    try {
+      await lease.connection.beginTransaction();
+      const value = await lease.operation.wait(callback(transaction));
+      await transaction.finish();
+      lease.operation.check();
+      committing = true;
+      await lease.connection.commit();
+      return value;
+    } catch (error) {
+      await transaction.finish().catch(() => {});
+      if (committing && !lease.connection.isValid()) {
+        throw new LosslessClientError('ambiguous_write', 'Transaction commit outcome is unknown.', { cause: error });
+      }
+      if (lease.operation.signal.aborted) throw lease.operation.signal.reason;
+      try { await lease.connection.rollback(); }
+      catch (rollbackError) { throw new AggregateError([error, rollbackError], 'Transaction and rollback failed.'); }
+      throw error;
+    } finally { await lease.close(); }
+  }
+
+  public close(): Promise<void> {
+    return this.closing ??= (async () => {
+      this.closed = true;
+      for (const operation of this.pendingOperations) operation.cancel();
+      await Promise.allSettled([...this.acquisitions]);
+      for (const lease of this.leases) lease.operation.cancel();
+      await Promise.all([...this.leases].map(lease => lease.close()));
+      await this.connecting?.catch(() => {});
+      const pool = this.pool;
+      this.pool = undefined;
+      await pool?.end();
+    })();
+  }
+}
+
+/** Bound to a single transaction callback and physical connection. */
+export class SqlTransaction {
+  private active?: Promise<unknown>;
+  private failure?: unknown;
+  private finished = false;
+  constructor(private readonly lease: ILease) {}
+
+  private run<Result>(action: () => Promise<Result>): Promise<Result> {
+    if (this.finished || this.active) return Promise.reject(new LosslessClientError('invalid_argument', 'Transaction is closed or already has an active operation.'));
+    if (this.failure) return Promise.reject(this.failure);
+    this.lease.operation.check();
+    const result = action().catch(error => { this.failure = error; throw error; });
+    this.active = result;
+    void result.finally(() => { if (this.active === result) this.active = undefined; }).catch(() => {});
+    return result;
+  }
+
+  public execute(statement: ISqlStatement): Promise<ISqlMutationResult> {
+    validateStatement(statement);
+    return this.run(async () => mutationResult(await this.lease.connection.execute(statement.sql, boundValues(statement))));
+  }
+
+  public query<Row = Record<string, unknown>>(statement: ISqlStatement, options: Pick<IQueryOptions, 'maxRows' | 'maxBytes'> = {}): Promise<Row[]> {
+    validateStatement(statement);
+    const limits = resultLimits(options);
+    return this.run(async () => {
+      const rows: Row[] = [];
+      let bytes = 0;
+      const source = this.lease.connection.queryStream(statement.sql, boundValues(statement));
+      try {
+        for await (const row of source) {
+          this.lease.operation.check();
+          bytes += valueBytes(row);
+          if (rows.length >= limits.rows || bytes > limits.bytes) {
+            throw new LosslessClientError('result_limit', 'Transaction result exceeded its materialization limit.');
+          }
+          rows.push(row as Row);
+        }
+        return rows;
+      } catch (error) {
+        this.lease.connection.destroy();
+        throw error;
+      } finally { source.destroy(); }
+    });
+  }
+
+  public async finish(): Promise<void> {
+    this.finished = true;
+    await this.active;
+    if (this.failure) throw this.failure;
+  }
+}

package/ts/sqldb/classes.sqltable.ts

@@ -0,0 +1,70 @@
+import type { SqlConnection } from './classes.sqlconnection.js';
+import type { IInsertOptions, IQueryOptions, ISqlStatement, TSqlValue } from './interfaces.js';
+import type { IOperationOptions } from '../core/interfaces.js';
+import { quoteIdentifier } from './interfaces.js';
+import { LosslessClientError } from '../core/classes.error.js';
+
+export type TSqlSelector<Row> = Partial<{ [Key in keyof Row]: TSqlValue }>;
+export interface ISqlSelectOptions<Row> extends IQueryOptions {
+  columns?: (keyof Row & string)[];
+  orderBy?: { column: keyof Row & string; direction?: 'ASC' | 'DESC' }[];
+  limit?: number;
+}
+
+/** A typed handle to an existing relational table. Schema creation is explicit SQL or a migration. */
+export class SqlTable<Row extends { [Key in keyof Row]: TSqlValue }> {
+  constructor(private readonly connection: SqlConnection, public readonly name: string) { quoteIdentifier(name); }
+
+  private where(selector: TSqlSelector<Row>, required = false): ISqlStatement {
+    const entries = Object.entries(selector);
+    if (required && !entries.length) throw new LosslessClientError('invalid_argument', 'A nonempty selector is required for table mutations.');
+    const values: TSqlValue[] = [];
+    const sql = entries.map(([key, value]) => {
+      if (value === undefined) throw new LosslessClientError('invalid_argument', 'Undefined selector values are not allowed.');
+      if (value === null) return quoteIdentifier(key) + ' IS NULL';
+      values.push(value as TSqlValue);
+      return quoteIdentifier(key) + ' = ?';
+    }).join(' AND ');
+    return { sql: sql ? ' WHERE ' + sql : '', values };
+  }
+
+  public selectStatement(selector: TSqlSelector<Row> = {}, options: ISqlSelectOptions<Row> = {}): ISqlStatement {
+    const where = this.where(selector);
+    if (options.columns?.length === 0) throw new LosslessClientError('invalid_argument', 'Select at least one column.');
+    let sql = 'SELECT ' + (options.columns?.map(quoteIdentifier).join(', ') ?? '*') + ' FROM ' + quoteIdentifier(this.name) + where.sql;
+    if (options.orderBy?.length) sql += ' ORDER BY ' + options.orderBy.map(order => {
+      const direction = order.direction ?? 'ASC';
+      if (!['ASC', 'DESC'].includes(direction)) throw new LosslessClientError('invalid_argument', 'Invalid sort direction.');
+      return quoteIdentifier(order.column) + ' ' + direction;
+    }).join(', ');
+    if (options.limit !== undefined) {
+      if (!Number.isSafeInteger(options.limit) || options.limit < 0) throw new LosslessClientError('invalid_argument', 'Limit must be a nonnegative safe integer.');
+      sql += ' LIMIT ' + options.limit;
+    }
+    return { sql, values: where.values };
+  }
+
+  public query(selector: TSqlSelector<Row> = {}, options: ISqlSelectOptions<Row> = {}): Promise<Row[]> {
+    return this.connection.query<Row>(this.selectStatement(selector, options), options);
+  }
+  public stream(selector: TSqlSelector<Row> = {}, options: ISqlSelectOptions<Row> = {}): AsyncGenerator<Row> {
+    return this.connection.stream<Row>(this.selectStatement(selector, options), options);
+  }
+  public insert(rows: Iterable<Row> | AsyncIterable<Row>, options: IInsertOptions = {}) {
+    return this.connection.insert(this.name, rows, options);
+  }
+  public update(selector: TSqlSelector<Row>, changes: Partial<Row>, options: IOperationOptions = {}) {
+    const where = this.where(selector, true);
+    const entries = Object.entries(changes);
+    if (!entries.length || entries.some(([, value]) => value === undefined)) {
+      throw new LosslessClientError('invalid_argument', 'Supply at least one defined changed value.');
+    }
+    return this.connection.execute({ sql: 'UPDATE ' + quoteIdentifier(this.name) + ' SET ' +
+      entries.map(([key]) => quoteIdentifier(key) + ' = ?').join(', ') + where.sql,
+      values: [...entries.map(([, value]) => value as TSqlValue), ...where.values!] }, options);
+  }
+  public delete(selector: TSqlSelector<Row>, options: IOperationOptions = {}) {
+    const where = this.where(selector, true);
+    return this.connection.execute({ sql: 'DELETE FROM ' + quoteIdentifier(this.name) + where.sql, values: where.values }, options);
+  }
+}

package/ts/sqldb/clickhouse/classes.metricwriter.ts

@@ -0,0 +1,72 @@
+import { valueBytes } from '../interfaces.js';
+import { LosslessClientError } from '../../core/classes.error.js';
+
+export interface IMetricWriterOptions { batchSize?: number; maxBatchBytes?: number; flushIntervalMs?: number }
+
+/** Await write() for backpressure. flush()/close() settle only after server acknowledgement. */
+export class MetricWriter<Row> {
+  private batch: Row[] = [];
+  private bytes = 0;
+  private active?: Promise<void>;
+  private failure?: unknown;
+  private closed = false;
+  private closing?: Promise<void>;
+  private timer?: ReturnType<typeof setTimeout>;
+  private readonly batchSize: number;
+  private readonly maxBytes: number;
+
+  constructor(private readonly send: (rows: Row[]) => Promise<void>, private readonly options: IMetricWriterOptions = {}) {
+    this.batchSize = options.batchSize ?? 500;
+    this.maxBytes = options.maxBatchBytes ?? 4 * 1024 * 1024;
+    for (const limit of [this.batchSize, this.maxBytes, options.flushIntervalMs ?? 1000]) {
+      if (!Number.isSafeInteger(limit) || limit < 1) throw new RangeError('Writer limits must be positive integers.');
+    }
+  }
+  private check(): void {
+    if (this.failure) throw this.failure;
+    if (this.closed) throw new LosslessClientError('closed', 'Metric writer is closed.');
+    if (this.active) throw new LosslessClientError('invalid_argument', 'Await the preceding writer operation before writing again.');
+  }
+  private perform(action: () => Promise<void>): Promise<void> {
+    const task = action().catch(error => { this.failure = error; throw error; });
+    this.active = task;
+    void task.finally(() => {
+      if (this.active === task) this.active = undefined;
+      this.schedule();
+    }).catch(() => {});
+    return task;
+  }
+  private async sendBatch(): Promise<void> {
+    clearTimeout(this.timer); this.timer = undefined;
+    if (!this.batch.length) return;
+    await this.send(this.batch);
+    this.batch = []; this.bytes = 0;
+  }
+  public write(row: Row): Promise<void> {
+    this.check();
+    const size = valueBytes(row);
+    if (size > this.maxBytes) throw new LosslessClientError('invalid_argument', 'Metric row exceeds maxBatchBytes.');
+    return this.perform(async () => {
+      if (this.batch.length && this.bytes + size > this.maxBytes) await this.sendBatch();
+      this.batch.push(structuredClone(row)); this.bytes += size;
+      if (this.batch.length >= this.batchSize) await this.sendBatch();
+    });
+  }
+  private schedule(): void {
+    if (this.timer || this.closed || this.failure || !this.batch.length) return;
+    this.timer = setTimeout(() => {
+      this.timer = undefined;
+      if (!this.closed && !this.failure && !this.active) void this.flush().catch(() => {});
+    }, this.options.flushIntervalMs ?? 1000);
+    this.timer.unref();
+  }
+  public flush(): Promise<void> { this.check(); return this.perform(() => this.sendBatch()); }
+  public close(): Promise<void> {
+    return this.closing ??= (async () => {
+      this.closed = true; clearTimeout(this.timer);
+      await this.active;
+      if (this.failure) throw this.failure;
+      await this.sendBatch();
+    })();
+  }
+}

package/ts/sqldb/clickhouse/index.ts

@@ -0,0 +1,16 @@
+// Core
+export * from './smartclickhouse.classes.smartclickhouse.js';
+export * from './smartclickhouse.classes.clickhousetable.js';
+export * from './smartclickhouse.classes.httpclient.js';
+
+// Query & Results
+export * from './smartclickhouse.classes.querybuilder.js';
+export * from './smartclickhouse.classes.resultset.js';
+
+// Time Data Table (backward compat)
+export * from './smartclickhouse.classes.timedatatable.js';
+
+// Types
+export * from './smartclickhouse.types.js';
+export { MetricWriter } from './classes.metricwriter.js';
+export type { IMetricWriterOptions } from './classes.metricwriter.js';