@edge-base/server 0.1.1

package/src/lib/log-writer.ts

@@ -0,0 +1,272 @@
+/**
+ * LogWriter adapter pattern.
+ *
+ * Provides environment-aware log storage with extended analytics fields:
+ * - Cloud:         AnalyticsEngineWriter — 10 blobs, 6 doubles
+ * - Docker/Self-hosted: SQLiteLogWriter — LogsDO with 3-tier pre-aggregation
+ * - Dev fallback:  ConsoleLogWriter — structured console output
+ *
+ * Usage:
+ *   const logger = createLogWriter(env, executionCtx);
+ *   logger.write({ method, path, status, duration, userId, category, ... });
+ *
+ * Data Point Layout (Analytics Engine):
+ *   index1:  userId (or 'anonymous')
+ *   blob1:   method        blob6:  subcategory
+ *   blob2:   path          blob7:  target1
+ *   blob3:   status (str)  blob8:  target2
+ *   blob4:   error         blob9:  operation
+ *   blob5:   category      blob10: region
+ *   double1: status        double4: requestSize
+ *   double2: duration      double5: responseSize
+ *   double3: timestamp     double6: resultCount
+ */
+
+// ─── Types ───
+
+export interface LogEntry {
+  // Core fields (original)
+  method: string;
+  path: string;
+  status: number;
+  duration: number;
+  userId?: string;
+  error?: string;
+  timestamp?: number;
+  // Route classification (from route-parser)
+  category?: string;
+  subcategory?: string;
+  target1?: string;
+  target2?: string;
+  operation?: string;
+  // Request context
+  region?: string;
+  requestSize?: number;
+  responseSize?: number;
+  resultCount?: number;
+}
+
+export interface QueryResult {
+  data: Record<string, unknown>[];
+}
+
+export interface LogWriter {
+  write(entry: LogEntry): void;
+  query(sql: string): Promise<QueryResult>;
+}
+
+function isDurableObjectResetError(err: unknown): boolean {
+  return (
+    typeof err === 'object'
+    && err !== null
+    && 'durableObjectReset' in err
+    && (err as { durableObjectReset?: unknown }).durableObjectReset === true
+  );
+}
+
+// ─── AnalyticsEngineWriter (Cloud) ───
+
+/**
+ * Log writer using Cloudflare Analytics Engine.
+ * Non-blocking fire-and-forget write. 10 blobs + 6 doubles per data point.
+ * SQL API queries are proxied via admin endpoint (requires CF account credentials).
+ */
+export class AnalyticsEngineWriter implements LogWriter {
+  constructor(private analytics: AnalyticsEngineDataset) {}
+
+  write(entry: LogEntry): void {
+    this.analytics.writeDataPoint({
+      indexes: [entry.userId || 'anonymous'],
+      blobs: [
+        entry.method,                      // blob1
+        entry.path,                        // blob2
+        String(entry.status),              // blob3
+        entry.error || '',                 // blob4
+        entry.category || '',              // blob5
+        entry.subcategory || '',           // blob6
+        entry.target1 || '',               // blob7
+        entry.target2 || '',               // blob8
+        entry.operation || '',             // blob9
+        entry.region || '',                // blob10
+      ],
+      doubles: [
+        entry.status,                      // double1
+        entry.duration,                    // double2
+        entry.timestamp || Date.now(),     // double3
+        entry.requestSize || 0,            // double4
+        entry.responseSize || 0,           // double5
+        entry.resultCount || 0,            // double6
+      ],
+    });
+  }
+
+  async query(_sql: string): Promise<QueryResult> {
+    // Analytics Engine SQL API requires account-level credentials
+    // and is called via the Cloudflare REST API, not from Workers.
+    // Admin endpoint /admin/api/data/analytics proxies these queries.
+    return { data: [] };
+  }
+}
+
+// ─── SQLiteLogWriter (Docker / Self-hosted) ───
+
+/**
+ * Log writer that buffers entries and flushes to LogsDO in batches.
+ * LogsDO stores logs in SQLite with 3-tier pre-aggregation:
+ *   _logs_raw (24h) → _logs_hourly (90d) → _logs_daily (permanent)
+ *
+ * Buffering: flushes at 50 entries or after 100ms, whichever comes first.
+ * Uses ExecutionContext.waitUntil() for non-blocking writes.
+ *
+ * Retry policy: failed batches are re-queued once (retried = true).
+ * If a retried batch fails again, entries fall back to console.
+ * Buffer is capped at 1000 entries to prevent memory exhaustion.
+ */
+export class SQLiteLogWriter implements LogWriter {
+  private static readonly MAX_BUFFER = 1000;
+  private buffer: (LogEntry & { _retried?: boolean })[] = [];
+  private flushScheduled = false;
+
+  constructor(
+    private logsDO: { fetch: (input: RequestInfo) => Promise<Response> },
+    private ctx?: { waitUntil: (promise: Promise<unknown>) => void },
+  ) {}
+
+  write(entry: LogEntry): void {
+    if (this.buffer.length >= SQLiteLogWriter.MAX_BUFFER) {
+      // Drop oldest to prevent memory exhaustion
+      this.buffer.shift();
+    }
+    this.buffer.push(entry);
+
+    if (this.buffer.length >= 50) {
+      // Immediate flush when buffer is full
+      const promise = this.flush();
+      if (this.ctx) {
+        this.ctx.waitUntil(promise);
+      }
+    } else if (!this.flushScheduled) {
+      // Schedule delayed flush for batching
+      this.flushScheduled = true;
+      const promise = this.scheduledFlush();
+      if (this.ctx) {
+        this.ctx.waitUntil(promise);
+      }
+    }
+  }
+
+  private async scheduledFlush(): Promise<void> {
+    await new Promise(r => setTimeout(r, 100));
+    this.flushScheduled = false;
+    await this.flush();
+  }
+
+  private async flush(): Promise<void> {
+    if (!this.buffer.length) return;
+    const batch = [...this.buffer];
+    this.buffer = [];
+
+    try {
+      await this.logsDO.fetch(
+        new Request('http://internal/internal/logs/write', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ entries: batch }),
+        }),
+      );
+    } catch (err) {
+      // Split batch: re-queue entries that haven't been retried yet
+      const retryable = batch.filter(e => !e._retried);
+      const dropped = batch.filter(e => e._retried);
+
+      if (retryable.length) {
+        for (const e of retryable) e._retried = true;
+        // Re-queue at front, respecting buffer cap
+        const space = SQLiteLogWriter.MAX_BUFFER - this.buffer.length;
+        this.buffer.unshift(...retryable.slice(0, space));
+      }
+
+      if (isDurableObjectResetError(err)) {
+        return;
+      }
+
+      if (dropped.length) {
+        console.error(`[EdgeBase] Failed to write ${dropped.length} logs to LogsDO (retry exhausted):`, err);
+        for (const entry of dropped) {
+          console.log(`[LOG] ${entry.method} ${entry.path} ${entry.status} ${entry.duration}ms`);
+        }
+      } else {
+        console.warn(`[EdgeBase] LogsDO write failed, ${retryable.length} entries queued for retry:`, err);
+      }
+    }
+  }
+
+  async query(sql: string): Promise<QueryResult> {
+    try {
+      const params = new URLSearchParams({ sql });
+      const resp = await this.logsDO.fetch(
+        new Request(`http://internal/internal/logs/query?${params}`),
+      );
+      return (await resp.json()) as QueryResult;
+    } catch {
+      return { data: [] };
+    }
+  }
+}
+
+// ─── ConsoleLogWriter (Dev fallback) ───
+
+/**
+ * Fallback log writer for local development.
+ * Outputs structured log lines to console. No persistence.
+ */
+export class ConsoleLogWriter implements LogWriter {
+  write(entry: LogEntry): void {
+    const ts = new Date(entry.timestamp || Date.now()).toISOString();
+    const cat = entry.category ? ` [${entry.category}]` : '';
+    const line = `[${ts}]${cat} ${entry.method} ${entry.path} ${entry.status} ${entry.duration}ms${entry.userId ? ` user=${entry.userId}` : ''}${entry.error ? ` error=${entry.error}` : ''}`;
+    if (entry.status >= 500) {
+      console.error(line);
+    } else if (entry.status >= 400) {
+      console.warn(line);
+    } else {
+      console.log(line);
+    }
+  }
+
+  async query(_sql: string): Promise<QueryResult> {
+    return { data: [] };
+  }
+}
+
+// ─── Factory ───
+
+/**
+ * Create the appropriate LogWriter based on environment bindings.
+ *
+ * Priority:
+ *   1. env.ANALYTICS exists → AnalyticsEngineWriter (Cloud)
+ *   2. env.LOGS exists      → SQLiteLogWriter (Docker / Self-hosted)
+ *   3. Otherwise            → ConsoleLogWriter (dev fallback)
+ */
+export function createLogWriter(
+  env: Record<string, unknown>,
+  executionCtx?: { waitUntil: (promise: Promise<unknown>) => void },
+): LogWriter {
+  // Cloud: Analytics Engine
+  const analytics = env.ANALYTICS as AnalyticsEngineDataset | undefined;
+  if (analytics) {
+    return new AnalyticsEngineWriter(analytics);
+  }
+
+  // Docker / Self-hosted: LogsDO SQLite
+  const logsNs = env.LOGS as { idFromName: (name: string) => { toString: () => string }; get: (id: unknown) => { fetch: (input: RequestInfo) => Promise<Response> } } | undefined;
+  if (logsNs) {
+    const logsId = logsNs.idFromName('logs:main');
+    const logsDO = logsNs.get(logsId);
+    return new SQLiteLogWriter(logsDO, executionCtx);
+  }
+
+  // Dev fallback
+  return new ConsoleLogWriter();
+}

package/src/lib/namespace-dump.ts

@@ -0,0 +1,125 @@
+import { EdgeBaseError, type EdgeBaseConfig } from '@edge-base/shared';
+import type { Env } from '../types.js';
+import { callDO, getDbDoName, getD1BindingName, shouldRouteToD1 } from './do-router.js';
+import { ensureD1Schema } from './d1-schema-init.js';
+import { ensurePgSchema } from './postgres-schema-init.js';
+import {
+  ensureLocalDevPostgresSchema,
+  getLocalDevPostgresExecOptions,
+  getProviderBindingName,
+  withPostgresConnection,
+} from './postgres-executor.js';
+
+interface DumpNamespaceTablesOptions {
+  includeMeta?: boolean;
+  tableNames?: string[];
+}
+
+export async function dumpNamespaceTables(
+  env: Env,
+  config: EdgeBaseConfig,
+  namespace: string,
+  options: DumpNamespaceTablesOptions = {},
+): Promise<Record<string, unknown[]>> {
+  const dbBlock = config.databases?.[namespace];
+  if (!dbBlock) {
+    throw new EdgeBaseError(404, `Namespace '${namespace}' not found in config.`);
+  }
+
+  const tableNames = options.tableNames?.length
+    ? options.tableNames
+    : Object.keys(dbBlock.tables ?? {});
+  const tables: Record<string, unknown[]> = {};
+  const includeMeta = options.includeMeta !== false;
+
+  if (dbBlock.provider === 'neon' || dbBlock.provider === 'postgres') {
+    const bindingName = getProviderBindingName(namespace);
+    const envRecord = env as unknown as Record<string, unknown>;
+    const hyperdrive = envRecord[bindingName] as { connectionString?: string } | undefined;
+    const envKey = dbBlock.connectionString ?? `${bindingName}_URL`;
+    const connStr = hyperdrive?.connectionString ?? (envRecord[envKey] as string | undefined);
+    if (!connStr) {
+      throw new EdgeBaseError(500, `PostgreSQL connection not available for '${namespace}'.`);
+    }
+
+    const localDevOptions = getLocalDevPostgresExecOptions(env as unknown as Record<string, unknown>, namespace);
+    if (localDevOptions) {
+      await ensureLocalDevPostgresSchema(localDevOptions);
+    }
+    await withPostgresConnection(connStr, async (query) => {
+      if (!localDevOptions) {
+        await ensurePgSchema(connStr, namespace, dbBlock.tables ?? {}, query);
+      }
+      for (const tableName of tableNames) {
+        try {
+          const result = await query(`SELECT * FROM "${tableName}"`, []);
+          tables[tableName] = result.rows;
+        } catch {
+          tables[tableName] = [];
+        }
+      }
+
+      if (includeMeta) {
+        try {
+          const meta = await query('SELECT * FROM "_meta"', []);
+          tables['_meta'] = meta.rows;
+        } catch {
+          tables['_meta'] = [];
+        }
+      }
+    }, localDevOptions);
+
+    return tables;
+  }
+
+  if (shouldRouteToD1(namespace, config)) {
+    const bindingName = getD1BindingName(namespace);
+    const db = (env as unknown as Record<string, unknown>)[bindingName] as D1Database | undefined;
+    if (!db) {
+      throw new EdgeBaseError(500, `D1 binding '${bindingName}' not available for '${namespace}'.`);
+    }
+
+    await ensureD1Schema(db, namespace, dbBlock.tables ?? {});
+    for (const tableName of tableNames) {
+      try {
+        const result = await db.prepare(`SELECT * FROM "${tableName}"`).all();
+        tables[tableName] = result.results ?? [];
+      } catch {
+        tables[tableName] = [];
+      }
+    }
+
+    if (includeMeta) {
+      try {
+        const meta = await db.prepare('SELECT * FROM "_meta"').all();
+        tables['_meta'] = meta.results ?? [];
+      } catch {
+        tables['_meta'] = [];
+      }
+    }
+
+    return tables;
+  }
+
+  const doName = getDbDoName(namespace);
+  const response = await callDO(env.DATABASE, doName, '/internal/backup/dump', {
+    headers: { 'X-DO-Name': doName },
+  });
+
+  if (!response.ok) {
+    throw new EdgeBaseError(response.status as 500, `Namespace dump failed: ${await response.text()}`);
+  }
+
+  const payload = (await response.json()) as {
+    tables?: Record<string, Array<Record<string, unknown>>>;
+  };
+
+  for (const tableName of tableNames) {
+    tables[tableName] = payload.tables?.[tableName] ?? [];
+  }
+  if (includeMeta) {
+    tables['_meta'] = payload.tables?._meta ?? [];
+  }
+
+  return tables;
+}