hide-a-bed 5.2.7 → 6.0.0-beta.0

package/impl/bulkGet.test.mts

@@ -0,0 +1,159 @@
+import assert from 'node:assert/strict'
+import test, { suite } from 'node:test'
+import needle from 'needle'
+import type { CouchConfigInput } from '../schema/config.mts'
+import { z } from 'zod'
+import { RetryableError } from './utils/errors.mts'
+import { bulkGet, bulkGetDictionary } from './bulkGet.mts'
+import { TEST_DB_URL } from '../test/setup-db.mts'
+
+const config: CouchConfigInput = {
+  couch: TEST_DB_URL
+}
+
+async function ensureDoc(id: string, body: Record<string, unknown>) {
+  await needle(
+    'put',
+    `${TEST_DB_URL}/${id}`,
+    {
+      _id: id,
+      ...body
+    },
+    { json: true }
+  )
+}
+
+suite('bulkGet', () => {
+  test('integration with pouchdb-server', async t => {
+    await ensureDoc('doc-1', { value: 42 })
+    await ensureDoc('doc-valid', { count: 7 })
+    await ensureDoc('doc-invalid', { count: 'nope' })
+
+    await t.test('fetches docs and not-found rows', async () => {
+      const response = await bulkGet(config, ['doc-1', 'doc-missing'])
+      assert.strictEqual(response.rows.length, 2)
+      const [first, second] = response.rows
+      assert.strictEqual(first?.id, 'doc-1')
+      assert.strictEqual(first?.doc?._id, 'doc-1')
+      assert.strictEqual(first?.doc?.value, 42)
+      assert.strictEqual(second?.error, 'not_found')
+      assert.strictEqual(second?.key, 'doc-missing')
+    })
+
+    await t.test('supports includeDocs=false via _bulkGetWithOptions', async () => {
+      const response = await bulkGet(config, ['doc-1'], {
+        includeDocs: false
+      })
+      assert.strictEqual(response.rows.length, 1)
+      const [row] = response.rows
+      assert.strictEqual(row?.id, 'doc-1')
+      assert.ok(row?.value?.rev)
+      assert.ok(!('doc' in (row as Record<string, unknown>)))
+    })
+
+    await t.test('validates documents when schema provided', async () => {
+      const schema = z.looseObject({
+        _id: z.string(),
+        _rev: z.string().optional(),
+        count: z.number()
+      })
+
+      const valid = await bulkGet(config, ['doc-valid'], {
+        validate: {
+          docSchema: schema
+        }
+      })
+      assert.strictEqual(valid.rows[0]?.doc?.count, 7)
+
+      await assert.rejects(
+        () =>
+          bulkGet(config, ['doc-invalid'], {
+            validate: {
+              docSchema: schema
+            }
+          }),
+        (err: unknown) => {
+          assert.ok(Array.isArray(err))
+          assert.match(err[0]?.message, /Invalid input:/)
+          return true
+        }
+      )
+    })
+
+    await t.test('skips invalid documents when onInvalidDoc=skip', async () => {
+      const schema = z.looseObject({
+        _id: z.string(),
+        _rev: z.string().optional(),
+        count: z.number()
+      })
+
+      const response = await bulkGet(config, ['doc-valid', 'doc-invalid'], {
+        validate: {
+          docSchema: schema,
+          onInvalidDoc: 'skip'
+        }
+      })
+
+      assert.strictEqual(response.rows.length, 1)
+      assert.strictEqual(response.rows[0]?.doc?._id, 'doc-valid')
+      assert.strictEqual(response.rows[0]?.doc?.count, 7)
+    })
+
+    await t.test('throws RetryableError for retryable status codes', async () => {
+      const offlineConfig: CouchConfigInput = {
+        couch: 'http://localhost:6553/offline-db'
+      }
+
+      await assert.rejects(
+        () => bulkGet(offlineConfig, ['doc-1']),
+        (err: unknown) => err instanceof RetryableError && err.statusCode === 503
+      )
+    })
+
+    await t.test('bulkGetDictionary groups results', async () => {
+      const result = await bulkGetDictionary(config, ['doc-valid', 'doc-missing'])
+      assert.deepStrictEqual(Object.keys(result.found), ['doc-valid'])
+      assert.strictEqual(result.found['doc-valid'].count, 7)
+      assert.deepStrictEqual(Object.keys(result.notFound), ['doc-missing'])
+      assert.strictEqual(result.notFound['doc-missing'].error, 'not_found')
+    })
+
+    await t.test('bulkGetDictionary with validation schema', async () => {
+      const schema = z.looseObject({
+        _id: z.string(),
+        _rev: z.string().optional(),
+        count: z.number()
+      })
+
+      const result = await bulkGetDictionary(config, ['doc-valid', 'doc-not-there'], {
+        validate: {
+          docSchema: schema
+        }
+      })
+
+      assert.deepStrictEqual(Object.keys(result.found), ['doc-valid'])
+      assert.strictEqual(result.found['doc-valid'].count, 7)
+      assert.deepStrictEqual(Object.keys(result.notFound), ['doc-not-there'])
+      assert.strictEqual(result.notFound['doc-not-there'].error, 'not_found')
+    })
+
+    await t.test('bulkGetDictionary skips invalid docs when requested', async () => {
+      const schema = z.looseObject({
+        _id: z.string(),
+        _rev: z.string().optional(),
+        count: z.number()
+      })
+
+      const result = await bulkGetDictionary(config, ['doc-valid', 'doc-invalid'], {
+        validate: {
+          docSchema: schema,
+          onInvalidDoc: 'skip'
+        }
+      })
+
+      assert.deepStrictEqual(Object.keys(result.found), ['doc-valid'])
+      assert.strictEqual(result.found['doc-valid'].count, 7)
+      assert.deepStrictEqual(Object.keys(result.notFound), [])
+    })
+  })
+})

package/impl/bulkRemove.mts

@@ -0,0 +1,98 @@
+import { bulkGet } from './bulkGet.mts'
+import { bulkSave } from './bulkSave.mts'
+import { createLogger } from './utils/logger.mts'
+import { remove } from './remove.mts'
+import { CouchDoc } from '../schema/couch/couch.output.schema.ts'
+import { CouchConfig, type CouchConfigInput } from '../schema/config.mts'
+
+/**
+ * Removes multiple documents from a CouchDB database using the _bulk_docs endpoint.
+ * It first retrieves the documents by their IDs, marks them as deleted, and then
+ * sends them back to the database for deletion.
+ *
+ * See https://docs.couchdb.org/en/stable/api/database/bulk-api.html#post--db-_bulk_docs
+ *
+ * @param configInput - The CouchDB configuration input.
+ * @param ids - An array of document IDs to be removed.
+ * @returns A promise that resolves to an array of results from the bulk delete operation.
+ *
+ * @example
+ * ```ts
+ * const config: CouchConfigInput = {
+ *   couch: 'http://localhost:5984/mydb',
+ *   useConsoleLogger: true
+ * };
+ * const idsToRemove = ['doc1', 'doc2', 'doc3'];
+ * const results = await bulkRemove(config, idsToRemove);
+ * console.log(results);
+ * ```
+ *
+ * @throws Will throw an error if the provided configuration is invalid or if the bulk delete operation fails.
+ */
+export const bulkRemove = async (configInput: CouchConfigInput, ids: string[]) => {
+  const config = CouchConfig.parse(configInput)
+  const logger = createLogger(config)
+  logger.info(`Starting bulk remove for ${ids.length} documents`)
+  const resp = await bulkGet(config, ids)
+  const toRemove: Array<CouchDoc> = []
+  resp.rows?.forEach(row => {
+    if (!row.doc) return
+    try {
+      const d = CouchDoc.parse(row.doc)
+      d._deleted = true
+      toRemove.push(d)
+    } catch (e) {
+      logger.warn(`Invalid document structure in bulk remove: ${row.id}`, e)
+    }
+  })
+  if (!toRemove.length) return []
+  const result = await bulkSave(config, toRemove)
+  return result
+}
+
+/**
+ * Removes multiple documents from a CouchDB database by their IDs using individual delete operations.
+ * It first retrieves the documents to get their revision IDs, then deletes each document one by one.
+ *
+ * See https://docs.couchdb.org/en/stable/api/document/common.html#delete--db-docid
+ *
+ * @param configInput - The CouchDB configuration input.
+ * @param ids - An array of document IDs to be removed.
+ * @returns A promise that resolves to an array of results from the individual delete operations.
+ *
+ * @example
+ * ```ts
+ * const config: CouchConfigInput = {
+ *   couch: 'http://localhost:5984/mydb',
+ *   useConsoleLogger: true
+ * };
+ * const idsToRemove = ['doc1', 'doc2', 'doc3'];
+ * const results = await bulkRemoveMap(config, idsToRemove);
+ * console.log(results);
+ * ```
+ *
+ * @throws Will throw an error if the provided configuration is invalid or if any delete operation fails.
+ */
+export const bulkRemoveMap = async (configInput: CouchConfigInput, ids: string[]) => {
+  const config = CouchConfig.parse(configInput)
+  const logger = createLogger(config)
+  logger.info(`Starting bulk remove map for ${ids.length} documents`)
+
+  const { rows } = await bulkGet(config, ids, { includeDocs: false })
+
+  const results = []
+  for (const row of rows || []) {
+    try {
+      if (!row.value?.rev) throw new Error(`no rev found for doc ${row.id}`)
+      if (!row.id) {
+        throw new Error(`no id found for doc ${row}`)
+      }
+
+      const result = await remove(config, row.id, row.value.rev)
+      results.push(result)
+    } catch (e) {
+      logger.warn(`Error removing a doc in bulk remove map: ${row.id}`, e)
+    }
+  }
+  return results
+}

package/impl/bulkRemove.test.mts

@@ -0,0 +1,102 @@
+import assert from 'node:assert/strict'
+import test, { suite } from 'node:test'
+import needle from 'needle'
+import type { CouchConfigInput } from '../schema/config.mts'
+import { bulkRemove, bulkRemoveMap } from './bulkRemove.mts'
+import { TEST_DB_URL } from '../test/setup-db.mts'
+
+const config: CouchConfigInput = {
+  couch: TEST_DB_URL
+}
+
+type DocBody = Record<string, unknown>
+
+async function saveDoc(id: string, body: DocBody) {
+  const response = await needle(
+    'put',
+    `${TEST_DB_URL}/${id}`,
+    {
+      _id: id,
+      ...body
+    },
+    { json: true }
+  )
+
+  if (response.statusCode !== 201 && response.statusCode !== 200) {
+    throw new Error(`Failed to save document ${id}: ${response.statusCode}`)
+  }
+
+  return response.body as { rev: string }
+}
+
+async function getDoc(id: string) {
+  return needle('get', `${TEST_DB_URL}/${id}`, null, { json: true })
+}
+
+suite('bulkRemove', () => {
+  test('it should throw if provided config is invalid', async () => {
+    await assert.rejects(async () => {
+      // @ts-expect-error testing invalid config
+      await bulkRemove({ notAnOption: true, couch: DB_URL, useConsoleLogger: true }, ['doc1'])
+    })
+
+    await assert.rejects(async () => {
+      // @ts-expect-error testing invalid config
+      await bulkRemoveMap({ anotherBadOption: 123, couch: DB_URL, useConsoleLogger: true }, [
+        'doc1'
+      ])
+    })
+  })
+
+  test('integration with pouchdb-server', async t => {
+    await t.test('removes documents via _bulk_docs', async () => {
+      await saveDoc('bulk-remove-doc-1', { kind: 'test', count: 1 })
+
+      const results = await bulkRemove(config, ['bulk-remove-doc-1'])
+      assert.strictEqual(results.length, 1)
+      const [first] = results
+      assert.strictEqual(first?.id, 'bulk-remove-doc-1')
+      assert.strictEqual(first?.ok, true)
+      assert.ok(typeof first?.rev === 'string')
+
+      const { statusCode, body } = await getDoc('bulk-remove-doc-1')
+      assert.strictEqual(statusCode, 404)
+      assert.strictEqual(body?.error, 'not_found')
+    })
+
+    await t.test('returns empty array when docs are missing', async () => {
+      const results = await bulkRemove(config, ['bulk-remove-missing'])
+      assert.deepStrictEqual(results, [])
+    })
+
+    await t.test('bulkRemoveMap removes each document individually', async () => {
+      await saveDoc('bulk-remove-map-doc-1', { kind: 'map', count: 1 })
+
+      const results = await bulkRemoveMap(config, ['bulk-remove-map-doc-1'])
+      assert.strictEqual(results.length, 1)
+      const [first] = results
+      assert.ok(first)
+      assert.strictEqual(first.id, 'bulk-remove-map-doc-1')
+      assert.strictEqual(first.ok, true)
+      assert.strictEqual(first.statusCode, 200)
+
+      const { statusCode, body } = await getDoc('bulk-remove-map-doc-1')
+      assert.strictEqual(statusCode, 404)
+      assert.strictEqual(body?.error, 'not_found')
+    })
+
+    await t.test('bulkRemoveMap skips docs without revs', async () => {
+      await saveDoc('bulk-remove-map-doc-2', { kind: 'map', count: 2 })
+
+      const results = await bulkRemoveMap(config, [
+        'bulk-remove-map-doc-2',
+        'bulk-remove-map-missing'
+      ])
+      assert.strictEqual(results.length, 1)
+      const [first] = results
+      assert.ok(first)
+      assert.strictEqual(first.id, 'bulk-remove-map-doc-2')
+      assert.strictEqual(first.ok, true)
+    })
+  })
+})

package/impl/bulkSave.mts

@@ -0,0 +1,286 @@
+import needle from 'needle'
+import { createLogger } from './utils/logger.mts'
+import { mergeNeedleOpts } from './utils/mergeNeedleOpts.mts'
+import { bulkGetDictionary } from './bulkGet.mts'
+import { setupEmitter } from './utils/trackedEmitter.mts'
+import {
+  TransactionSetupError,
+  TransactionVersionConflictError,
+  TransactionBulkOperationError,
+  TransactionRollbackError
+} from './utils/transactionErrors.mts'
+import {
+  BulkSaveResponse,
+  CouchDoc,
+  type CouchDocInput
+} from '../schema/couch/couch.output.schema.ts'
+import type { CouchConfigInput } from '../schema/config.mts'
+import { RetryableError } from './utils/errors.mts'
+import { withRetry } from './retry.mts'
+import { put } from './put.mts'
+
+/**
+ * Bulk saves documents to CouchDB using the _bulk_docs endpoint.
+ *
+ * @see
+ * https://docs.couchdb.org/en/stable/api/database/bulk-api.html#db-bulk-docs
+ *
+ * @param {CouchConfigInput} config - The CouchDB configuration.
+ * @param {CouchDocInput[]} docs - An array of documents to save.
+ * @returns {Promise<BulkSaveResponse>} - The response from CouchDB after the bulk save operation.
+ *
+ * @throws {RetryableError} When a retryable HTTP status code is encountered or no response is received.
+ * @throws {Error} When CouchDB returns a non-retryable error payload.
+ */
+export const bulkSave = async (config: CouchConfigInput, docs: CouchDocInput[]) => {
+  const logger = createLogger(config)
+
+  if (docs == null || !docs.length) {
+    logger.error('bulkSave called with no docs')
+    throw new Error('no docs provided')
+  }
+
+  logger.info(`Starting bulk save of ${docs.length} documents`)
+  const url = `${config.couch}/_bulk_docs`
+  const body = { docs }
+  const opts = {
+    json: true,
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  }
+  const mergedOpts = mergeNeedleOpts(config, opts)
+  let resp
+  try {
+    resp = await needle('post', url, body, mergedOpts)
+  } catch (err) {
+    logger.error('Network error during bulk save:', err)
+    RetryableError.handleNetworkError(err)
+  }
+  if (!resp) {
+    logger.error('No response received from bulk save request')
+    throw new RetryableError('no response', 503)
+  }
+  if (RetryableError.isRetryableStatusCode(resp.statusCode)) {
+    logger.warn(`Retryable status code received: ${resp.statusCode}`)
+    throw new RetryableError('retryable error during bulk save', resp.statusCode)
+  }
+  if (resp.statusCode !== 201) {
+    logger.error(`Unexpected status code: ${resp.statusCode}`)
+    throw new Error('could not save')
+  }
+  const results = resp?.body || []
+  return BulkSaveResponse.parse(results)
+}
+
+type TransactionStatus = 'pending' | 'completed' | 'rolled_back' | 'rollback_failed'
+type TransactionDoc = {
+  _id: string
+  _rev: string | null | undefined
+  type: 'transaction'
+  status: TransactionStatus
+  changes: CouchDocInput[]
+  timestamp: string
+}
+
+/**
+ * Performs a bulk save of documents within a transaction context.
+ *
+ * @remarks
+ * This operation ensures that either all documents are saved successfully, or none are, maintaining data consistency.
+ * If any document fails to save, the operation will attempt to roll back all changes.
+ *
+ * The transactionId has to be unique for the lifetime of the app. It is used to prevent two processes from executing the same transaction. It is up to you to craft a transactionId that uniquely represents this transaction, and that also is the same if another process tries to generate it.
+ *
+ * Exceptions to handle:
+ *
+ * `TransactionSetupError` Thrown if the transaction document cannot be created. Usually because it already exists
+ * `TransactionVersionConflictError` Thrown if there are version conflicts with existing documents.
+ * `TransactionBulkOperationError` Thrown if the bulk save operation fails for some documents.
+ * `TransactionRollbackError` Thrown if the rollback operation fails after a transaction failure.
+ *
+ * @example
+ * ```ts
+ * const docsToSave = [
+ *  { _id: 'doc1', foo: 'bar' },
+ *  { _id: 'doc2', foo: 'baz' }
+ * ];
+ *
+ * try {
+ *  const results = await bulkSaveTransaction(config, 'unique-transaction-id', docsToSave);
+ *  console.log('Bulk save successful:', results);
+ *  } catch (error) {
+ *  console.error('Bulk save transaction failed:', error);
+ * }
+ * ```
+ *
+ * @param {CouchConfigInput} config - The CouchDB configuration.
+ * @param {string} transactionId - A unique identifier for the transaction.
+ * @param {CouchDocInput[]} docs - An array of documents to save.
+ * @returns {Promise<BulkSaveResponse>} - The transaction save results.
+ * @throws {TransactionSetupError} When the transaction document cannot be created.
+ * @throws {TransactionVersionConflictError} When there are version conflicts with existing documents.
+ * @throws {TransactionBulkOperationError} When the bulk save operation fails for some documents.
+ * @throws {TransactionRollbackError} When the rollback operation fails after a transaction failure.
+ */
+export const bulkSaveTransaction = async (
+  config: CouchConfigInput,
+  transactionId: string,
+  docs: CouchDocInput[]
+): Promise<BulkSaveResponse> => {
+  const emitter = setupEmitter(config)
+  const logger = createLogger(config)
+  const retryOptions = {
+    maxRetries: config.maxRetries ?? 10,
+    initialDelay: config.initialDelay ?? 1000,
+    backoffFactor: config.backoffFactor ?? 2
+  }
+  const _put = config.bindWithRetry
+    ? withRetry(put.bind(null, config), retryOptions)
+    : put.bind(null, config)
+  logger.info(`Starting bulk save transaction ${transactionId} for ${docs.length} documents`)
+
+  // Create transaction document
+  const transactionDoc: TransactionDoc = {
+    _id: `txn:${transactionId}`,
+    _rev: null,
+    type: 'transaction',
+    status: 'pending',
+    changes: docs,
+    timestamp: new Date().toISOString()
+  }
+
+  // Save transaction document
+  let transactionResponse = await _put(transactionDoc)
+  logger.debug('Transaction document created:', transactionDoc, transactionResponse)
+  await emitter.emit('transaction-created', {
+    transactionResponse,
+    txnDoc: transactionDoc
+  })
+  if (transactionResponse.error) {
+    throw new TransactionSetupError('Failed to create transaction document', {
+      error: transactionResponse.error,
+      response: transactionResponse
+    })
+  }
+
+  // Get current revisions of all documents
+  const existingDocs = await bulkGetDictionary(
+    config,
+    docs.map(d => d._id)
+  )
+  logger.debug('Fetched current revisions of documents:', existingDocs)
+  await emitter.emit('transaction-revs-fetched', existingDocs)
+
+  /** @type {string[]} */
+  const revErrors: string[] = []
+  // if any of the existingDocs, and the docs provided do not match on rev, then throw an error
+  docs.forEach(d => {
+    if (!d._id) return
+    if (existingDocs.found[d._id] && existingDocs.found[d._id]._rev !== d._rev)
+      revErrors.push(d._id)
+    if (existingDocs.notFound[d._id] && d._rev) revErrors.push(d._id)
+  })
+
+  if (revErrors.length > 0) {
+    throw new TransactionVersionConflictError(revErrors)
+  }
+  logger.debug('Checked document revisions:', existingDocs)
+  await emitter.emit('transaction-revs-checked', existingDocs)
+
+  const providedDocsById: Record<string, CouchDocInput> = {}
+  docs.forEach(d => {
+    if (!d._id) return
+    providedDocsById[d._id] = d
+  })
+
+  const newDocsToRollback: BulkSaveResponse = []
+  const potentialExistingDocsToRollback: BulkSaveResponse = []
+  const failedDocs: BulkSaveResponse = []
+
+  try {
+    logger.info('Transaction started:', transactionDoc)
+    await emitter.emit('transaction-started', transactionDoc)
+    // Apply updates
+    const results = await bulkSave(config, docs)
+    logger.info('Transaction updates applied:', results)
+    await emitter.emit('transaction-updates-applied', results)
+
+    // Check for failures
+    results.forEach(r => {
+      if (!r.id) return // not enough info
+      if (!r.error) {
+        if (existingDocs.notFound[r.id]) newDocsToRollback.push(r)
+        if (existingDocs.found[r.id]) potentialExistingDocsToRollback.push(r)
+      } else {
+        failedDocs.push(r)
+      }
+    })
+    if (failedDocs.length > 0) {
+      throw new TransactionBulkOperationError(failedDocs)
+    }
+
+    // Update transaction status to completed
+    transactionDoc.status = 'completed'
+    transactionDoc._rev = transactionResponse.rev
+    transactionResponse = await _put(transactionDoc)
+    logger.info('Transaction completed:', transactionDoc)
+    await emitter.emit('transaction-completed', {
+      transactionResponse,
+      transactionDoc
+    })
+    if (transactionResponse.statusCode !== 201) {
+      logger.error('Failed to update transaction status to completed')
+    }
+
+    return results
+  } catch (error) {
+    logger.error('Transaction failed, attempting rollback:', error)
+
+    // Rollback changes
+    const toRollback: CouchDoc[] = []
+    potentialExistingDocsToRollback.forEach(row => {
+      if (!row.id || !row.rev) return
+      const doc = existingDocs.found[row.id]
+      doc._rev = row.rev
+      toRollback.push(doc)
+    })
+    newDocsToRollback.forEach(d => {
+      if (!d.id || !d.rev) return
+      const before = JSON.parse(JSON.stringify(providedDocsById[d.id]))
+      before._rev = d.rev
+      before._deleted = true
+      toRollback.push(before)
+    })
+
+    // rollback all the changes
+    const bulkRollbackResult = await bulkSave(config, toRollback)
+    let status: TransactionStatus = 'rolled_back'
+    bulkRollbackResult.forEach(r => {
+      if (r.error) status = 'rollback_failed'
+    })
+    logger.warn('Transaction rolled back:', { bulkRollbackResult, status })
+    await emitter.emit('transaction-rolled-back', {
+      bulkRollbackResult,
+      status
+    })
+
+    // Update transaction status to rolled back
+    transactionDoc.status = status
+    transactionDoc._rev = transactionResponse.rev || null
+    transactionResponse = await _put(transactionDoc)
+    logger.warn('Transaction rollback status updated:', transactionDoc)
+    await emitter.emit('transaction-rolled-back-status', {
+      transactionResponse,
+      transactionDoc
+    })
+    if (transactionResponse.statusCode !== 201) {
+      logger.error('Failed to update transaction status to rolled_back')
+    }
+    throw new TransactionRollbackError(
+      'Transaction failed and rollback was unsuccessful',
+      error as Error,
+      bulkRollbackResult
+    )
+  }
+}