@shipfox/api-secrets 2.0.0

package/src/db/variables.ts

@@ -0,0 +1,187 @@
+import {and, eq, inArray, isNull, sql} from 'drizzle-orm';
+import {SecretBatchScopeMismatchError} from '#core/errors.js';
+import {db, type Tx} from './db.js';
+import {type SecretVariable, secretVariables, toSecretVariable} from './schema/variables.js';
+import {
+  lookupWithPrecedenceWhere,
+  normalizedProjectId,
+  type StoreScope,
+  scopeConflictTargetWhere,
+  scopeExactWhere,
+} from './scope.js';
+
+export interface SecretVariableWrite {
+  workspaceId: string;
+  projectId: string | null;
+  namespace: string;
+  key: string;
+  value: string;
+  lastEditedBy?: string | null | undefined;
+}
+
+export async function getSecretVariableRowWithPrecedence(
+  params: StoreScope & {workspaceId: string; namespace: string; key: string},
+  tx?: Tx,
+): Promise<SecretVariable | undefined> {
+  const executor = tx ?? db();
+  const rows = await executor
+    .select()
+    .from(secretVariables)
+    .where(
+      lookupWithPrecedenceWhere(
+        {
+          workspaceId: secretVariables.workspaceId,
+          projectId: secretVariables.projectId,
+          namespace: secretVariables.namespace,
+          key: secretVariables.key,
+        },
+        params,
+      ),
+    )
+    .orderBy(sql`${secretVariables.projectId} NULLS LAST`)
+    .limit(1);
+  const row = rows[0];
+  return row ? toSecretVariable(row) : undefined;
+}
+
+export async function listSecretVariableRowsByNamespace(
+  params: StoreScope & {workspaceId: string; namespace: string},
+  tx?: Tx,
+): Promise<SecretVariable[]> {
+  const executor = tx ?? db();
+  const rows = await executor.execute<SecretVariableRow>(sql`
+    SELECT DISTINCT ON (key)
+      id, workspace_id, project_id, namespace, key, value,
+      created_at, updated_at, last_edited_by
+    FROM secrets_variables
+    WHERE ${listByNamespaceSql(params)}
+    ORDER BY key, project_id NULLS LAST, id
+  `);
+  return rows.rows.map(rowToSecretVariable);
+}
+
+export async function upsertSecretVariableRows(rows: SecretVariableWrite[], tx: Tx): Promise<void> {
+  if (rows.length === 0) return;
+  const projectId = batchProjectId(rows);
+  await tx
+    .insert(secretVariables)
+    .values(rows)
+    .onConflictDoUpdate({
+      target: projectId
+        ? [
+            secretVariables.workspaceId,
+            secretVariables.projectId,
+            secretVariables.namespace,
+            secretVariables.key,
+          ]
+        : [secretVariables.workspaceId, secretVariables.namespace, secretVariables.key],
+      targetWhere: scopeConflictTargetWhere({projectId}),
+      set: {
+        value: sql`excluded.value`,
+        lastEditedBy: sql`excluded.last_edited_by`,
+        updatedAt: sql`NOW()`,
+      },
+    });
+}
+
+export async function countSecretVariableRows(
+  params: StoreScope & {workspaceId: string; namespace: string; keys: string[]},
+  tx: Tx,
+): Promise<number> {
+  if (params.keys.length === 0) return 0;
+
+  const rows = await tx
+    .select({count: sql<string | number>`COUNT(*)`})
+    .from(secretVariables)
+    .where(
+      and(
+        eq(secretVariables.workspaceId, params.workspaceId),
+        eq(secretVariables.namespace, params.namespace),
+        scopeExactWhere(
+          {
+            workspaceId: secretVariables.workspaceId,
+            projectId: secretVariables.projectId,
+            namespace: secretVariables.namespace,
+            key: secretVariables.key,
+          },
+          params,
+        ) ?? isNull(secretVariables.projectId),
+        inArray(secretVariables.key, params.keys),
+      ),
+    );
+
+  return Number(rows[0]?.count ?? 0);
+}
+
+export async function deleteSecretVariableRows(
+  params: StoreScope & {workspaceId: string; namespace: string; keys?: string[] | undefined},
+  tx?: Tx,
+): Promise<number> {
+  if (params.keys && params.keys.length === 0) return 0;
+
+  const executor = tx ?? db();
+  const filters = [
+    eq(secretVariables.workspaceId, params.workspaceId),
+    eq(secretVariables.namespace, params.namespace),
+    scopeExactWhere(
+      {
+        workspaceId: secretVariables.workspaceId,
+        projectId: secretVariables.projectId,
+        namespace: secretVariables.namespace,
+        key: secretVariables.key,
+      },
+      params,
+    ) ?? isNull(secretVariables.projectId),
+  ];
+  if (params.keys && params.keys.length > 0) {
+    filters.push(inArray(secretVariables.key, params.keys));
+  }
+  const deleted = await executor
+    .delete(secretVariables)
+    .where(and(...filters))
+    .returning({id: secretVariables.id});
+  return deleted.length;
+}
+
+function batchProjectId(rows: SecretVariableWrite[]): string | null {
+  const projectId = normalizedProjectId({projectId: rows[0]?.projectId});
+  const mixedScope = rows.some(
+    (row) => normalizedProjectId({projectId: row.projectId}) !== projectId,
+  );
+  if (mixedScope) throw new SecretBatchScopeMismatchError();
+  return projectId;
+}
+
+function listByNamespaceSql(params: StoreScope & {workspaceId: string; namespace: string}) {
+  const projectId = normalizedProjectId(params);
+  if (!projectId) {
+    return sql`workspace_id = ${params.workspaceId} AND namespace = ${params.namespace} AND project_id IS NULL`;
+  }
+  return sql`workspace_id = ${params.workspaceId} AND namespace = ${params.namespace} AND (project_id = ${projectId} OR project_id IS NULL)`;
+}
+
+interface SecretVariableRow extends Record<string, unknown> {
+  id: string;
+  workspace_id: string;
+  project_id: string | null;
+  namespace: string;
+  key: string;
+  value: string;
+  created_at: Date;
+  updated_at: Date;
+  last_edited_by: string | null;
+}
+
+function rowToSecretVariable(row: SecretVariableRow): SecretVariable {
+  return {
+    id: row.id,
+    workspaceId: row.workspace_id,
+    projectId: row.project_id,
+    namespace: row.namespace,
+    key: row.key,
+    value: row.value,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    lastEditedBy: row.last_edited_by,
+  };
+}

package/src/index.test.ts

@@ -0,0 +1,11 @@
+import {describe, expect, it} from '@shipfox/vitest/vi';
+import {SecretValueTooLargeError} from './index.js';
+
+describe('package exports', () => {
+  it('exports the oversized secret value error from the package root', () => {
+    const error = new SecretValueTooLargeError(64 * 1024);
+
+    expect(error).toBeInstanceOf(Error);
+    expect(error.maxBytes).toBe(64 * 1024);
+  });
+});

package/src/index.ts

@@ -0,0 +1,50 @@
+import {secretsEventSchemas} from '@shipfox/api-secrets-dto';
+import type {ShipfoxModule} from '@shipfox/node-module';
+import {db, migrationsPath, secretsOutbox} from '#db/index.js';
+import {registerSecretsServiceMetrics} from '#metrics/index.js';
+import {secretsE2eRoutes} from '#presentation/e2eRoutes/index.js';
+import {secretsRoutes} from '#presentation/routes/index.js';
+
+export {
+  BUILTIN_LOCAL_STORE,
+  DekUnwrapError,
+  DekWrapError,
+  deleteManagedSecret,
+  deleteManagedVariable,
+  deleteSecrets,
+  deleteVariables,
+  getManagedVariable,
+  getSecret,
+  getSecretsByNamespace,
+  getVariable,
+  getVariablesByNamespace,
+  KekConfigurationError,
+  KekVersionStrandedError,
+  listManagedSecrets,
+  listManagedVariables,
+  NamespaceValidationError,
+  resolveSecretStore,
+  rotateWorkspaceDataKeys,
+  SecretBatchScopeMismatchError,
+  SecretDecryptionError,
+  SecretKeyValidationError,
+  SecretNotFoundError,
+  type SecretStoreProvider,
+  SecretValueTooLargeError,
+  setManagedSecrets,
+  setManagedVariables,
+  setSecrets,
+  setVariables,
+  UnknownSecretStoreError,
+  VariableNotFoundError,
+  WorkspaceSecretCapExceededError,
+} from '#core/index.js';
+
+export const secretsModule: ShipfoxModule = {
+  name: 'secrets',
+  database: {db, migrationsPath},
+  routes: secretsRoutes,
+  e2eRoutes: [secretsE2eRoutes],
+  metrics: registerSecretsServiceMetrics,
+  publishers: [{name: 'secrets', table: secretsOutbox, db, eventSchemas: secretsEventSchemas}],
+};

package/src/metrics/index.ts

@@ -0,0 +1,19 @@
+export {
+  classifyDekAccessError,
+  classifyKekRotationError,
+  classifySecretsOperationError,
+  operationScope,
+  recordSecretsDekAccess,
+  recordSecretsEntriesMutated,
+  recordSecretsKekRotation,
+  recordSecretsOperation,
+  type SecretsDekAccessOutcome,
+  type SecretsKekRotationOutcome,
+  type SecretsMetricOperation,
+  type SecretsMetricResource,
+  type SecretsMetricScope,
+  type SecretsMetricSurface,
+  type SecretsMutationEffect,
+  type SecretsOperationOutcome,
+} from './instance.js';
+export {registerSecretsServiceMetrics} from './service.js';

package/src/metrics/instance.ts

@@ -0,0 +1,219 @@
+import {instanceMetrics} from '@shipfox/node-opentelemetry';
+import {
+  DekUnwrapError,
+  DekWrapError,
+  KekVersionStrandedError,
+  NamespaceValidationError,
+  SecretBatchDuplicateKeyError,
+  SecretBatchScopeMismatchError,
+  SecretDecryptionError,
+  SecretKeyValidationError,
+  SecretNotFoundError,
+  SecretValueTooLargeError,
+  UnknownSecretStoreError,
+  VariableNotFoundError,
+  WorkspaceSecretCapExceededError,
+} from '#core/errors.js';
+
+const meter = instanceMetrics.getMeter('secrets');
+
+export type SecretsMetricResource = 'secret' | 'variable';
+export type SecretsMetricOperation = 'get' | 'get_namespace' | 'set' | 'delete' | 'list';
+export type SecretsMetricSurface = 'internal' | 'management';
+export type SecretsMetricScope = 'workspace' | 'project';
+export type SecretsOperationOutcome =
+  | 'success'
+  | 'not_found'
+  | 'validation_failed'
+  | 'cap_exceeded'
+  | 'value_too_large'
+  | 'decryption_failed'
+  | 'crypto_failed'
+  | 'failure';
+export type SecretsMutationEffect = 'created' | 'updated' | 'deleted';
+export type SecretsDekAccessOutcome =
+  | 'cache_hit'
+  | 'cache_expired'
+  | 'db_unwrapped'
+  | 'generated'
+  | 'unwrap_failed'
+  | 'wrap_failed'
+  | 'failure';
+export type SecretsKekRotationOutcome =
+  | 'rotated'
+  | 'skipped_current'
+  | 'skipped_race'
+  | 'none'
+  | 'stranded'
+  | 'failure';
+
+const operationCount = meter.createCounter<{
+  resource: SecretsMetricResource;
+  operation: SecretsMetricOperation;
+  surface: SecretsMetricSurface;
+  scope: SecretsMetricScope;
+  outcome: SecretsOperationOutcome;
+}>('secrets_operation', {
+  description: 'Secrets module operation attempts by resource, surface, scope, and outcome',
+});
+
+const operationDuration = meter.createHistogram<{
+  resource: SecretsMetricResource;
+  operation: SecretsMetricOperation;
+  surface: SecretsMetricSurface;
+  scope: SecretsMetricScope;
+  outcome: SecretsOperationOutcome;
+}>('secrets_operation_duration', {
+  description: 'Secrets module operation duration',
+  unit: 'ms',
+  advice: {explicitBucketBoundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000]},
+});
+
+const entriesMutatedCount = meter.createCounter<{
+  resource: SecretsMetricResource;
+  operation: 'set' | 'delete';
+  effect: SecretsMutationEffect;
+  surface: SecretsMetricSurface;
+}>('secrets_entries_mutated', {
+  description: 'Secret and variable entries mutated by operation, effect, and surface',
+});
+
+const dekAccessCount = meter.createCounter<{outcome: SecretsDekAccessOutcome}>(
+  'secrets_dek_access',
+  {description: 'Plaintext data-encryption key access attempts by outcome'},
+);
+
+const dekAccessDuration = meter.createHistogram<{outcome: SecretsDekAccessOutcome}>(
+  'secrets_dek_access_duration',
+  {
+    description: 'Plaintext data-encryption key access duration',
+    unit: 'ms',
+    advice: {explicitBucketBoundaries: [0, 1, 5, 10, 25, 50, 100, 250, 500, 1000]},
+  },
+);
+
+const kekRotationCount = meter.createCounter<{outcome: SecretsKekRotationOutcome}>(
+  'secrets_kek_rotation',
+  {description: 'Workspace data-key KEK rotation outcomes'},
+);
+
+const kekRotationDuration = meter.createHistogram<{outcome: SecretsKekRotationOutcome}>(
+  'secrets_kek_rotation_duration',
+  {
+    description: 'Workspace data-key KEK rotation duration',
+    unit: 'ms',
+    advice: {explicitBucketBoundaries: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000]},
+  },
+);
+
+function recordMetric(record: () => void): void {
+  try {
+    record();
+  } catch {
+    // Metrics must not affect secret storage outcomes.
+  }
+}
+
+export function operationScope(params: {
+  projectId?: string | null | undefined;
+}): SecretsMetricScope {
+  return params.projectId ? 'project' : 'workspace';
+}
+
+export function classifySecretsOperationError(error: unknown): SecretsOperationOutcome {
+  if (error instanceof SecretNotFoundError || error instanceof VariableNotFoundError) {
+    return 'not_found';
+  }
+  if (error instanceof WorkspaceSecretCapExceededError) return 'cap_exceeded';
+  if (error instanceof SecretValueTooLargeError) return 'value_too_large';
+  if (error instanceof SecretDecryptionError) return 'decryption_failed';
+  if (error instanceof DekUnwrapError || error instanceof DekWrapError) return 'crypto_failed';
+  if (
+    error instanceof NamespaceValidationError ||
+    error instanceof SecretKeyValidationError ||
+    error instanceof SecretBatchDuplicateKeyError ||
+    error instanceof SecretBatchScopeMismatchError ||
+    error instanceof UnknownSecretStoreError
+  ) {
+    return 'validation_failed';
+  }
+  return 'failure';
+}
+
+export function recordSecretsOperation(params: {
+  resource: SecretsMetricResource;
+  operation: SecretsMetricOperation;
+  surface: SecretsMetricSurface;
+  scope: SecretsMetricScope;
+  outcome: SecretsOperationOutcome;
+  durationMs: number;
+}): void {
+  recordMetric(() => {
+    const labels = {
+      resource: params.resource,
+      operation: params.operation,
+      surface: params.surface,
+      scope: params.scope,
+      outcome: params.outcome,
+    };
+    operationCount.add(1, labels);
+    operationDuration.record(params.durationMs, labels);
+  });
+}
+
+export function recordSecretsEntriesMutated(params: {
+  resource: SecretsMetricResource;
+  operation: 'set' | 'delete';
+  effect: SecretsMutationEffect;
+  surface: SecretsMetricSurface;
+  count: number;
+}): void {
+  if (params.count <= 0) return;
+
+  recordMetric(() =>
+    entriesMutatedCount.add(params.count, {
+      resource: params.resource,
+      operation: params.operation,
+      effect: params.effect,
+      surface: params.surface,
+    }),
+  );
+}
+
+export function recordSecretsDekAccess(params: {
+  outcome: SecretsDekAccessOutcome;
+  durationMs: number;
+}): void {
+  recordMetric(() => {
+    dekAccessCount.add(1, {outcome: params.outcome});
+    dekAccessDuration.record(params.durationMs, {outcome: params.outcome});
+  });
+}
+
+export function classifyDekAccessError(error: unknown): SecretsDekAccessOutcome {
+  if (error instanceof DekUnwrapError) return 'unwrap_failed';
+  if (error instanceof DekWrapError) return 'wrap_failed';
+  return 'failure';
+}
+
+export function recordSecretsKekRotation(params: {
+  outcome: SecretsKekRotationOutcome;
+  count?: number | undefined;
+  durationMs?: number | undefined;
+}): void {
+  const count = params.count ?? 1;
+  if (count <= 0 && params.durationMs === undefined) return;
+
+  recordMetric(() => {
+    if (count > 0) kekRotationCount.add(count, {outcome: params.outcome});
+    if (params.durationMs !== undefined) {
+      kekRotationDuration.record(params.durationMs, {outcome: params.outcome});
+    }
+  });
+}
+
+export function classifyKekRotationError(error: unknown): SecretsKekRotationOutcome {
+  if (error instanceof KekVersionStrandedError) return 'stranded';
+  if (error instanceof DekUnwrapError || error instanceof DekWrapError) return 'failure';
+  return 'failure';
+}

package/src/metrics/service.ts

@@ -0,0 +1,19 @@
+import {getServiceMetricsProvider} from '@shipfox/node-opentelemetry';
+import {countPublicEntriesByResource} from '#db/index.js';
+
+export function registerSecretsServiceMetrics(): void {
+  const meter = getServiceMetricsProvider().getMeter('secrets');
+
+  const publicEntries = meter.createObservableGauge('secrets_public_entries', {
+    description: 'Public secret and variable entries currently stored, excluding system namespaces',
+  });
+
+  meter.addBatchObservableCallback(
+    async (observer) => {
+      const counts = await countPublicEntriesByResource();
+      observer.observe(publicEntries, counts.secrets, {resource: 'secret'});
+      observer.observe(publicEntries, counts.variables, {resource: 'variable'});
+    },
+    [publicEntries],
+  );
+}

package/src/presentation/dto/index.ts

@@ -0,0 +1 @@
+export {toSecretDto, toVariableDto, toVariableListItemDto} from './management.js';

package/src/presentation/dto/management.ts

@@ -0,0 +1,36 @@
+import type {SecretDto, VariableDto, VariableListItemDto} from '@shipfox/api-secrets-dto';
+import type {SecretManagementRow, VariableManagementListRow} from '#db/index.js';
+import type {SecretVariable} from '#db/schema/variables.js';
+
+export function toSecretDto(secret: SecretManagementRow): SecretDto {
+  return {
+    key: secret.key,
+    project_id: secret.projectId,
+    created_at: secret.createdAt.toISOString(),
+    updated_at: secret.updatedAt.toISOString(),
+    last_edited_by: secret.lastEditedBy,
+  };
+}
+
+export function toVariableDto(variable: SecretVariable): VariableDto {
+  return {
+    key: variable.key,
+    project_id: variable.projectId,
+    value: variable.value,
+    created_at: variable.createdAt.toISOString(),
+    updated_at: variable.updatedAt.toISOString(),
+    last_edited_by: variable.lastEditedBy,
+  };
+}
+
+export function toVariableListItemDto(variable: VariableManagementListRow): VariableListItemDto {
+  return {
+    key: variable.key,
+    project_id: variable.projectId,
+    value: variable.value,
+    value_truncated: variable.valueTruncated,
+    created_at: variable.createdAt.toISOString(),
+    updated_at: variable.updatedAt.toISOString(),
+    last_edited_by: variable.lastEditedBy,
+  };
+}

package/src/presentation/e2eRoutes/create-secret.ts

@@ -0,0 +1,31 @@
+import {e2eCreateSecretBodySchema, e2eCreateSecretResponseSchema} from '@shipfox/api-secrets-dto';
+import {defineRoute} from '@shipfox/node-fastify';
+import {setManagedSecrets} from '#core/index.js';
+import {toSecretDto} from '#presentation/dto/index.js';
+import {translateManagementError} from '#presentation/routes/errors.js';
+
+type ManagedSecret = Awaited<ReturnType<typeof setManagedSecrets>>[number];
+
+export const createE2eSecretRoute = defineRoute({
+  method: 'POST',
+  path: '/secret',
+  description: 'Create a workspace secret for E2E tests.',
+  schema: {
+    body: e2eCreateSecretBodySchema,
+    response: {201: e2eCreateSecretResponseSchema},
+  },
+  errorHandler: translateManagementError,
+  handler: async (request, reply) => {
+    const secret = (
+      await setManagedSecrets({
+        workspaceId: request.body.workspace_id,
+        actorId: request.body.actor_id,
+        projectId: request.body.project_id,
+        entries: [{key: request.body.key, value: request.body.value}],
+      })
+    )[0] as ManagedSecret;
+
+    reply.code(201);
+    return toSecretDto(secret);
+  },
+});

package/src/presentation/e2eRoutes/create-variable.ts

@@ -0,0 +1,34 @@
+import {
+  e2eCreateVariableBodySchema,
+  e2eCreateVariableResponseSchema,
+} from '@shipfox/api-secrets-dto';
+import {defineRoute} from '@shipfox/node-fastify';
+import {setManagedVariables} from '#core/index.js';
+import {toVariableDto} from '#presentation/dto/index.js';
+import {translateManagementError} from '#presentation/routes/errors.js';
+
+type ManagedVariable = Awaited<ReturnType<typeof setManagedVariables>>[number];
+
+export const createE2eVariableRoute = defineRoute({
+  method: 'POST',
+  path: '/variable',
+  description: 'Create a workspace variable for E2E tests.',
+  schema: {
+    body: e2eCreateVariableBodySchema,
+    response: {201: e2eCreateVariableResponseSchema},
+  },
+  errorHandler: translateManagementError,
+  handler: async (request, reply) => {
+    const variable = (
+      await setManagedVariables({
+        workspaceId: request.body.workspace_id,
+        actorId: request.body.actor_id,
+        projectId: request.body.project_id,
+        entries: [{key: request.body.key, value: request.body.value}],
+      })
+    )[0] as ManagedVariable;
+
+    reply.code(201);
+    return toVariableDto(variable);
+  },
+});