@useatlas/create 0.0.6 → 0.0.7

package/templates/nextjs-standalone/src/lib/learn/suggestions.ts

@@ -0,0 +1,139 @@
+/**
+ * Query suggestion analysis engine.
+ *
+ * Reads the audit log, groups queries by normalized SQL fingerprint,
+ * scores by frequency + recency, and upserts into query_suggestions.
+ * Idempotent via INSERT … ON CONFLICT DO UPDATE.
+ */
+
+import { normalizeSQL, fingerprintSQL, extractPatternInfo, getYamlPatterns } from "@atlas/api/lib/learn/pattern-analyzer";
+import { getAuditLogQueries, upsertSuggestion } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+
+const log = createLogger("suggestions");
+
+const MIN_PATTERN_COUNT = 2;
+
+// ── Scoring ────────────────────────────────────────────────────────
+
+/** Score a suggestion by frequency weighted by recency (30-day half-life). */
+export function scoreSuggestion(frequency: number, lastSeenAt: Date): number {
+  const daysSince = (Date.now() - lastSeenAt.getTime()) / (1000 * 60 * 60 * 24);
+  return frequency * (1 / (1 + daysSince / 30));
+}
+
+// ── Grouping ───────────────────────────────────────────────────────
+
+interface AuditRow {
+  sql: string;
+  tables_accessed: string | null;
+  timestamp: string;
+}
+
+export interface GroupedPattern {
+  normalizedSql: string;
+  fingerprint: string;
+  count: number;
+  lastSeen: Date;
+  tables: string[];
+  primaryTable: string | null;
+  description: string;
+}
+
+/** Group audit rows by normalized SQL fingerprint. Exported for testing only. */
+export function _groupAuditRows(rows: AuditRow[]): Map<string, GroupedPattern> {
+  const groups = new Map<string, GroupedPattern>();
+
+  for (const row of rows) {
+    if (!row.sql) continue;
+    const normalized = normalizeSQL(row.sql);
+    if (!normalized) continue;
+    const fp = fingerprintSQL(normalized);
+
+    const existing = groups.get(fp);
+    const rowTime = new Date(row.timestamp);
+
+    if (existing) {
+      existing.count++;
+      if (rowTime > existing.lastSeen) {
+        existing.lastSeen = rowTime;
+      }
+    } else {
+      const info = extractPatternInfo(row.sql);
+      let tables: string[] = [];
+      try {
+        tables = row.tables_accessed ? JSON.parse(row.tables_accessed) : [];
+      } catch {
+        // intentionally ignored: malformed tables_accessed JSON
+      }
+      if (tables.length === 0 && info?.tables) {
+        tables = info.tables;
+      }
+
+      groups.set(fp, {
+        normalizedSql: normalized,
+        fingerprint: fp,
+        count: 1,
+        lastSeen: rowTime,
+        tables,
+        primaryTable: info?.primaryTable ?? tables[0] ?? null,
+        description: info?.description ?? "Query pattern",
+      });
+    }
+  }
+
+  return groups;
+}
+
+// ── Batch generation ───────────────────────────────────────────────
+
+/** Batch-generate suggestions from audit log. Idempotent via upsert. */
+export async function generateSuggestions(
+  orgId: string | null
+): Promise<{ created: number; updated: number }> {
+  const rows = await getAuditLogQueries(orgId);
+  if (rows.length === 0) {
+    log.info({ orgId }, "No audit log entries found for suggestion generation");
+    return { created: 0, updated: 0 };
+  }
+
+  const groups = _groupAuditRows(rows);
+
+  // Filter: drop low-frequency patterns
+  const candidates = [...groups.values()].filter((g) => g.count >= MIN_PATTERN_COUNT);
+
+  // Filter: drop patterns that exist in YAML query_patterns
+  let yamlPatterns: Set<string> | null = null;
+  try {
+    yamlPatterns = getYamlPatterns();
+  } catch {
+    // intentionally ignored: YAML patterns unavailable (SaaS mode)
+  }
+
+  const filtered = yamlPatterns
+    ? candidates.filter((c) => !yamlPatterns!.has(c.normalizedSql))
+    : candidates;
+
+  let created = 0;
+  let updated = 0;
+
+  for (const pattern of filtered) {
+    const score = scoreSuggestion(pattern.count, pattern.lastSeen);
+    const result = await upsertSuggestion({
+      orgId,
+      description: pattern.description,
+      patternSql: pattern.normalizedSql,
+      normalizedHash: pattern.fingerprint,
+      tablesInvolved: pattern.tables,
+      primaryTable: pattern.primaryTable,
+      frequency: pattern.count,
+      score,
+      lastSeenAt: pattern.lastSeen,
+    });
+    if (result === "created") created++;
+    else if (result === "updated") updated++;
+  }
+
+  log.info({ orgId, created, updated, total: filtered.length }, "Suggestion generation complete");
+  return { created, updated };
+}

package/templates/nextjs-standalone/src/lib/linear/store.ts

@@ -0,0 +1,200 @@
+/**
+ * Linear installation storage.
+ *
+ * Stores per-workspace Linear API keys in the internal database.
+ * Each workspace admin enters their own API key (BYOT).
+ */
+
+import { hasInternalDB, internalQuery } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+import type { LinearInstallation, LinearInstallationWithSecret } from "@atlas/api/lib/integrations/types";
+
+export type { LinearInstallation, LinearInstallationWithSecret } from "@atlas/api/lib/integrations/types";
+
+const log = createLogger("linear-store");
+
+// ---------------------------------------------------------------------------
+// Shared row parser
+// ---------------------------------------------------------------------------
+
+function parseInstallationRow(
+  row: Record<string, unknown>,
+  context: Record<string, unknown>,
+): LinearInstallationWithSecret | null {
+  const userId = row.user_id;
+  const apiKey = row.api_key;
+  if (typeof userId !== "string" || !userId || typeof apiKey !== "string" || !apiKey) {
+    log.warn(context, "Invalid Linear installation record in database");
+    return null;
+  }
+  return {
+    user_id: userId,
+    api_key: apiKey,
+    user_name: typeof row.user_name === "string" ? row.user_name : null,
+    user_email: typeof row.user_email === "string" ? row.user_email : null,
+    org_id: typeof row.org_id === "string" ? row.org_id : null,
+    installed_at: typeof row.installed_at === "string" ? row.installed_at : new Date().toISOString(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Read operations
+// ---------------------------------------------------------------------------
+
+/**
+ * Get the Linear installation for a user ID (Linear viewer ID).
+ */
+export async function getLinearInstallation(
+  userId: string,
+): Promise<LinearInstallationWithSecret | null> {
+  if (!hasInternalDB()) {
+    return null;
+  }
+
+  try {
+    const rows = await internalQuery<Record<string, unknown>>(
+      "SELECT user_id, api_key, user_name, user_email, org_id, installed_at::text FROM linear_installations WHERE user_id = $1",
+      [userId],
+    );
+    if (rows.length > 0) {
+      return parseInstallationRow(rows[0], { userId });
+    }
+    return null;
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), userId },
+      "Failed to query linear_installations",
+    );
+    throw err;
+  }
+}
+
+/**
+ * Get the Linear installation for an org. Returns null if not found or
+ * if no internal database is configured.
+ */
+export async function getLinearInstallationByOrg(
+  orgId: string,
+): Promise<LinearInstallation | null> {
+  if (!hasInternalDB()) {
+    return null;
+  }
+
+  try {
+    const rows = await internalQuery<Record<string, unknown>>(
+      "SELECT user_id, api_key, user_name, user_email, org_id, installed_at::text FROM linear_installations WHERE org_id = $1",
+      [orgId],
+    );
+    if (rows.length > 0) {
+      const full = parseInstallationRow(rows[0], { orgId });
+      if (!full) return null;
+      const { api_key: _, ...pub } = full;
+      return pub;
+    }
+    return null;
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), orgId },
+      "Failed to query linear_installations by org",
+    );
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Write operations
+// ---------------------------------------------------------------------------
+
+/**
+ * Save or update a Linear installation (API key submission).
+ * Throws if the Linear user is already bound to a different organization (hijack protection).
+ * Throws if the database write fails.
+ */
+export async function saveLinearInstallation(
+  userId: string,
+  opts: { orgId?: string; userName?: string; userEmail?: string; apiKey: string },
+): Promise<void> {
+  if (!hasInternalDB()) {
+    throw new Error("Cannot save Linear installation — no internal database configured");
+  }
+
+  const orgId = opts.orgId ?? null;
+  const userName = opts.userName ?? null;
+  const userEmail = opts.userEmail ?? null;
+
+  try {
+    // Atomic upsert with hijack protection — the WHERE clause rejects rows
+    // bound to a different org in one statement (no TOCTOU race).
+    const rows = await internalQuery<{ user_id: string }>(
+      `INSERT INTO linear_installations (user_id, api_key, user_name, user_email, org_id)
+       VALUES ($1, $2, $3, $4, $5)
+       ON CONFLICT (user_id) DO UPDATE SET
+         api_key = $2,
+         user_name = COALESCE($3, linear_installations.user_name),
+         user_email = COALESCE($4, linear_installations.user_email),
+         org_id = COALESCE($5, linear_installations.org_id),
+         installed_at = now()
+       WHERE linear_installations.org_id IS NULL OR linear_installations.org_id = $5
+       RETURNING user_id`,
+      [userId, opts.apiKey, userName, userEmail, orgId],
+    );
+
+    if (rows.length === 0) {
+      throw new Error(
+        `Linear user ${userId} is already bound to a different organization. ` +
+        `Disconnect the existing installation first.`,
+      );
+    }
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), userId },
+      "Failed to save linear_installations",
+    );
+    throw err;
+  }
+}
+
+/**
+ * Remove a Linear installation by user ID.
+ * Throws if no internal DB or if the query fails.
+ */
+export async function deleteLinearInstallation(userId: string): Promise<void> {
+  if (!hasInternalDB()) {
+    throw new Error("Cannot delete Linear installation — no internal database configured");
+  }
+
+  try {
+    await internalQuery("DELETE FROM linear_installations WHERE user_id = $1", [userId]);
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), userId },
+      "Failed to delete linear_installations",
+    );
+    throw err;
+  }
+}
+
+/**
+ * Remove the Linear installation for an org.
+ * Returns true if a row was deleted, false if no matching row found.
+ * Throws if no internal DB or if the query fails.
+ */
+export async function deleteLinearInstallationByOrg(orgId: string): Promise<boolean> {
+  if (!hasInternalDB()) {
+    throw new Error("Cannot delete Linear installation — no internal database configured");
+  }
+
+  try {
+    const rows = await internalQuery<{ user_id: string }>(
+      "DELETE FROM linear_installations WHERE org_id = $1 RETURNING user_id",
+      [orgId],
+    );
+    return rows.length > 0;
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), orgId },
+      "Failed to delete linear_installations by org",
+    );
+    throw err;
+  }
+}

package/templates/nextjs-standalone/src/lib/logger.ts

@@ -35,9 +35,9 @@ export function getRequestContext(): RequestContext | undefined {
 
 const isDev = process.env.NODE_ENV !== "production";
 
-// Redaction covers top-level fields and one-level nested (*.field) plus
-// array element access ([*].field). fast-redact does not support ** glob
-// wildcards, so deeper nesting would require additional explicit paths.
+// Redaction covers top-level fields, one-level nested (*.field), array
+// element access ([*].field), and known deep structures. fast-redact does
+// not support ** glob wildcards, so deep paths must be listed explicitly.
 export const redactPaths = [
   "connectionString",
   "databaseUrl",
@@ -45,18 +45,33 @@ export const redactPaths = [
   "password",
   "secret",
   "authorization",
+  "url",
   "*.connectionString",
   "*.databaseUrl",
   "*.apiKey",
   "*.password",
   "*.secret",
   "*.authorization",
+  "*.url",
   "[*].connectionString",
   "[*].databaseUrl",
   "[*].apiKey",
   "[*].password",
   "[*].secret",
   "[*].authorization",
+  "[*].url",
+  // Deep structures: datasource config, connection registry, plugin config
+  "datasources.*.url",
+  "datasources.*.connectionString",
+  "datasources.*.password",
+  "config.datasources.*.url",
+  "config.datasources.*.connectionString",
+  "connection.url",
+  "connection.connectionString",
+  "connection.password",
+  "connections.*.url",
+  "connections.*.connectionString",
+  "connections.*.password",
 ];
 
 const rootLogger = pino({
@@ -96,3 +111,20 @@ export function getLogger(): pino.Logger {
 export function createLogger(component: string): pino.Logger {
   return rootLogger.child({ component });
 }
+
+const VALID_LOG_LEVELS = new Set(["trace", "debug", "info", "warn", "error", "fatal"]);
+
+/**
+ * Update the root logger level at runtime.
+ *
+ * Used by the settings hot-reload system to apply ATLAS_LOG_LEVEL changes
+ * in SaaS mode without a server restart. Pino propagates the level change
+ * to all child loggers automatically.
+ *
+ * @returns true if the level was applied, false if the level is invalid.
+ */
+export function setLogLevel(level: string): boolean {
+  if (!VALID_LOG_LEVELS.has(level)) return false;
+  rootLogger.level = level;
+  return true;
+}

package/templates/nextjs-standalone/src/lib/metering.ts

@@ -0,0 +1,272 @@
+/**
+ * Usage metering helpers.
+ *
+ * Provides event logging and aggregate rollup for per-workspace usage
+ * tracking. Events are recorded to usage_events; summaries are
+ * materialized into usage_summaries on demand.
+ *
+ * Two write strategies:
+ * - `logUsageEvent` is fire-and-forget via `internalExecute` — async
+ *   errors are handled by internalExecute's circuit-breaker `.catch`.
+ * - `aggregateUsageSummary` awaits the query but swallows errors.
+ * In both cases, metering failures never propagate to the caller.
+ */
+
+import { createLogger } from "@atlas/api/lib/logger";
+import {
+  hasInternalDB,
+  internalExecute,
+  internalQuery,
+} from "@atlas/api/lib/db/internal";
+
+const log = createLogger("metering");
+
+// ---------------------------------------------------------------------------
+// Event types
+// ---------------------------------------------------------------------------
+
+// NOTE: "login" events are not yet emitted — active_users will be 0
+// until auth hooks emit login events. Query and token events are
+// emitted from the agent loop's onFinish callback.
+export type UsageEventType = "query" | "token" | "login";
+
+export interface UsageEvent {
+  workspaceId: string | null;
+  userId: string | null;
+  eventType: UsageEventType;
+  quantity: number;
+  metadata?: Record<string, unknown>;
+}
+
+// ---------------------------------------------------------------------------
+// Event logging (fire-and-forget)
+// ---------------------------------------------------------------------------
+
+/**
+ * Log a usage event. Fire-and-forget — async errors are handled by
+ * internalExecute's circuit-breaker. No-op if internal DB is not configured.
+ */
+export function logUsageEvent(event: UsageEvent): void {
+  if (!hasInternalDB()) return;
+
+  // Error handling is delegated to internalExecute's .catch handler,
+  // which logs failures with SQL context and trips the circuit breaker
+  // after 5 consecutive failures. No try/catch needed here — hasInternalDB()
+  // guards against the only synchronous throw path (DATABASE_URL unset).
+  internalExecute(
+    `INSERT INTO usage_events (workspace_id, user_id, event_type, quantity, metadata)
+     VALUES ($1, $2, $3, $4, $5)`,
+    [
+      event.workspaceId ?? null,
+      event.userId ?? null,
+      event.eventType,
+      event.quantity,
+      event.metadata ? JSON.stringify(event.metadata) : null,
+    ],
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Aggregation
+// ---------------------------------------------------------------------------
+
+/**
+ * Aggregate usage events into a summary row for the given workspace and period.
+ * Uses atomic upsert (ON CONFLICT ... DO UPDATE) for concurrent safety.
+ *
+ * @param workspaceId - The workspace/org to aggregate for.
+ * @param period - "daily" or "monthly".
+ * @param periodStart - The start of the period (truncated to day/month).
+ */
+export async function aggregateUsageSummary(
+  workspaceId: string,
+  period: "daily" | "monthly",
+  periodStart: Date,
+): Promise<void> {
+  if (!hasInternalDB()) return;
+
+  const periodStartISO = periodStart.toISOString();
+  const interval = period === "daily" ? "1 day" : "1 month";
+
+  try {
+    await internalQuery(
+      `INSERT INTO usage_summaries (workspace_id, period, period_start, query_count, token_count, active_users, storage_bytes)
+       SELECT
+         $1,
+         $2,
+         $3::timestamptz,
+         COALESCE(SUM(CASE WHEN event_type = 'query' THEN quantity ELSE 0 END), 0),
+         COALESCE(SUM(CASE WHEN event_type = 'token' THEN quantity ELSE 0 END), 0),
+         COALESCE(COUNT(DISTINCT CASE WHEN event_type = 'login' THEN user_id END), 0),
+         0
+       FROM usage_events
+       WHERE workspace_id = $1
+         AND created_at >= $3::timestamptz
+         AND created_at < $3::timestamptz + $4::interval
+       ON CONFLICT (workspace_id, period, period_start) DO UPDATE SET
+         query_count = EXCLUDED.query_count,
+         token_count = EXCLUDED.token_count,
+         active_users = EXCLUDED.active_users,
+         updated_at = now()`,
+      [workspaceId, period, periodStartISO, interval],
+    );
+  } catch (err) {
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err), workspaceId, period },
+      "Failed to aggregate usage summary",
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Query helpers (for admin API)
+// ---------------------------------------------------------------------------
+
+export interface UsageSummaryRow {
+  id: string;
+  workspace_id: string;
+  period: string;
+  period_start: string;
+  query_count: number;
+  token_count: number;
+  active_users: number;
+  storage_bytes: number;
+  updated_at: string;
+  [key: string]: unknown;
+}
+
+export interface UsageCurrentPeriod {
+  queryCount: number;
+  tokenCount: number;
+  activeUsers: number;
+  periodStart: string;
+  periodEnd: string;
+}
+
+/**
+ * Get the current period summary for a workspace by querying usage_events
+ * directly (real-time, not from pre-aggregated summaries).
+ */
+export async function getCurrentPeriodUsage(
+  workspaceId: string,
+): Promise<UsageCurrentPeriod> {
+  if (!hasInternalDB()) {
+    return { queryCount: 0, tokenCount: 0, activeUsers: 0, periodStart: "", periodEnd: "" };
+  }
+
+  const now = new Date();
+  const periodStart = new Date(now.getFullYear(), now.getMonth(), 1);
+  const periodEnd = new Date(now.getFullYear(), now.getMonth() + 1, 1);
+
+  const rows = await internalQuery<{
+    query_count: number;
+    token_count: number;
+    active_users: number;
+  }>(
+    `SELECT
+       COALESCE(SUM(CASE WHEN event_type = 'query' THEN quantity ELSE 0 END), 0)::int AS query_count,
+       COALESCE(SUM(CASE WHEN event_type = 'token' THEN quantity ELSE 0 END), 0)::int AS token_count,
+       COALESCE(COUNT(DISTINCT CASE WHEN event_type = 'login' THEN user_id END), 0)::int AS active_users
+     FROM usage_events
+     WHERE workspace_id = $1
+       AND created_at >= $2
+       AND created_at < $3`,
+    [workspaceId, periodStart.toISOString(), periodEnd.toISOString()],
+  );
+
+  const row = rows[0];
+  return {
+    queryCount: row?.query_count ?? 0,
+    tokenCount: row?.token_count ?? 0,
+    activeUsers: row?.active_users ?? 0,
+    periodStart: periodStart.toISOString(),
+    periodEnd: periodEnd.toISOString(),
+  };
+}
+
+/**
+ * Get historical usage summaries for a workspace.
+ */
+export async function getUsageHistory(
+  workspaceId: string,
+  period: "daily" | "monthly",
+  startDate?: string,
+  endDate?: string,
+  limit = 90,
+): Promise<UsageSummaryRow[]> {
+  if (!hasInternalDB()) return [];
+
+  const params: unknown[] = [workspaceId, period];
+  const conditions = ["workspace_id = $1", "period = $2"];
+
+  if (startDate) {
+    params.push(startDate);
+    conditions.push(`period_start >= $${params.length}`);
+  }
+  if (endDate) {
+    params.push(endDate);
+    conditions.push(`period_start <= $${params.length}`);
+  }
+
+  params.push(limit);
+  const limitIdx = params.length;
+
+  return internalQuery<UsageSummaryRow>(
+    `SELECT * FROM usage_summaries
+     WHERE ${conditions.join(" AND ")}
+     ORDER BY period_start DESC
+     LIMIT $${limitIdx}`,
+    params,
+  );
+}
+
+/**
+ * Get per-user usage breakdown within a workspace for a time range.
+ */
+export async function getUsageBreakdown(
+  workspaceId: string,
+  startDate?: string,
+  endDate?: string,
+  limit = 100,
+): Promise<Array<{
+  user_id: string;
+  query_count: number;
+  token_count: number;
+  login_count: number;
+}>> {
+  if (!hasInternalDB()) return [];
+
+  const params: unknown[] = [workspaceId];
+  const conditions = ["workspace_id = $1", "user_id IS NOT NULL"];
+
+  if (startDate) {
+    params.push(startDate);
+    conditions.push(`created_at >= $${params.length}`);
+  }
+  if (endDate) {
+    params.push(endDate);
+    conditions.push(`created_at <= $${params.length}`);
+  }
+
+  params.push(limit);
+  const limitIdx = params.length;
+
+  return internalQuery<{
+    user_id: string;
+    query_count: number;
+    token_count: number;
+    login_count: number;
+  }>(
+    `SELECT
+       user_id,
+       SUM(CASE WHEN event_type = 'query' THEN quantity ELSE 0 END)::int AS query_count,
+       SUM(CASE WHEN event_type = 'token' THEN quantity ELSE 0 END)::int AS token_count,
+       SUM(CASE WHEN event_type = 'login' THEN quantity ELSE 0 END)::int AS login_count
+     FROM usage_events
+     WHERE ${conditions.join(" AND ")}
+     GROUP BY user_id
+     ORDER BY query_count DESC
+     LIMIT $${limitIdx}`,
+    params,
+  );
+}