@useatlas/create 0.0.5 → 0.0.7

package/templates/docker/src/lib/learn/pattern-cache.ts

@@ -0,0 +1,229 @@
+/**
+ * In-memory cache for approved learned patterns with TTL-based expiry
+ * and keyword-based relevance filtering.
+ *
+ * Cache is keyed by orgId (null for global). Invalidated by admin
+ * approve/reject actions via `invalidatePatternCache()`.
+ */
+
+import { getApprovedPatterns, type ApprovedPatternRow } from "@atlas/api/lib/db/internal";
+import { getConfig } from "@atlas/api/lib/config";
+import { createLogger } from "@atlas/api/lib/logger";
+
+const log = createLogger("pattern-cache");
+
+// ---------------------------------------------------------------------------
+// Cache
+// ---------------------------------------------------------------------------
+
+const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
+const MAX_ENTRIES = 500;
+
+interface CacheEntry {
+  patterns: ApprovedPatternRow[];
+  expiresAt: number;
+  lastAccessedAt: number;
+}
+
+const cache = new Map<string, CacheEntry>();
+
+/** Canonical cache key — `"__global__"` for null orgId, prefixed to avoid collision. */
+function cacheKey(orgId: string | null): string {
+  return orgId === null ? "__global__" : `org:${orgId}`;
+}
+
+/** Get approved patterns for an org, hitting cache first.
+ *  DB failures are logged and return [] without being cached. */
+async function getCachedPatterns(orgId: string | null): Promise<ApprovedPatternRow[]> {
+  const key = cacheKey(orgId);
+  const entry = cache.get(key);
+
+  if (entry && Date.now() < entry.expiresAt) {
+    entry.lastAccessedAt = Date.now();
+    return entry.patterns;
+  }
+
+  try {
+    const patterns = await getApprovedPatterns(orgId);
+
+    // Evict oldest entry if cache is at capacity
+    if (cache.size >= MAX_ENTRIES) {
+      let oldestKey: string | undefined;
+      let oldestTime = Infinity;
+      for (const [k, v] of cache) {
+        if (v.lastAccessedAt < oldestTime) {
+          oldestTime = v.lastAccessedAt;
+          oldestKey = k;
+        }
+      }
+      if (oldestKey) cache.delete(oldestKey);
+    }
+
+    const now = Date.now();
+    cache.set(key, { patterns, expiresAt: now + DEFAULT_TTL_MS, lastAccessedAt: now });
+    return patterns;
+  } catch (err) {
+    log.warn(
+      { orgId, err: err instanceof Error ? err.message : String(err) },
+      "Failed to load approved patterns (table may not exist yet) — not caching",
+    );
+    return [];
+  }
+}
+
+/** Invalidate the pattern cache for a specific org. */
+export function invalidatePatternCache(orgId: string | null): void {
+  cache.delete(cacheKey(orgId));
+}
+
+/** Reset entire cache. For testing only. */
+export function _resetPatternCache(): void {
+  cache.clear();
+}
+
+// ---------------------------------------------------------------------------
+// Keyword extraction & relevance scoring
+// ---------------------------------------------------------------------------
+
+/** Common SQL/English stop words to ignore during keyword matching. */
+const STOP_WORDS = new Set([
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "shall",
+  "should", "may", "might", "must", "can", "could", "and", "but", "or",
+  "nor", "not", "so", "yet", "for", "of", "in", "on", "at", "to", "by",
+  "with", "from", "as", "into", "about", "between", "through", "after",
+  "before", "above", "below", "up", "down", "out", "off", "over", "under",
+  "then", "than", "that", "this", "these", "those", "what", "which", "who",
+  "whom", "how", "when", "where", "why", "all", "each", "every", "both",
+  "few", "more", "most", "other", "some", "such", "no", "only", "same",
+  "it", "its", "i", "me", "my", "we", "us", "our", "you", "your", "he",
+  "him", "his", "she", "her", "they", "them", "their",
+  // SQL keywords
+  "select", "from", "where", "join", "left", "right", "inner", "outer",
+  "on", "group", "order", "limit", "offset", "having", "union", "case",
+  "when", "else", "end", "as", "count", "sum", "avg", "min", "max",
+  "distinct", "null", "true", "false", "asc", "desc", "between", "like",
+  "insert", "update", "delete", "create", "alter", "drop", "table",
+  "index", "values", "set", "into", "not", "exists", "if", "and", "or",
+]);
+
+/** Extract meaningful keywords from a text string. */
+export function extractKeywords(text: string): Set<string> {
+  const words = text
+    .toLowerCase()
+    .replace(/[^a-z0-9_]/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length > 1 && !STOP_WORDS.has(w));
+  return new Set(words);
+}
+
+/** Score a pattern's relevance to a set of question keywords. */
+function scorePattern(
+  pattern: ApprovedPatternRow,
+  questionKeywords: Set<string>,
+): number {
+  const patternText = [
+    pattern.pattern_sql,
+    pattern.description ?? "",
+    pattern.source_entity ?? "",
+  ].join(" ");
+
+  const patternKeywords = extractKeywords(patternText);
+  let overlap = 0;
+  for (const kw of questionKeywords) {
+    if (patternKeywords.has(kw)) overlap++;
+  }
+  return overlap;
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/** Default maximum patterns to inject into the system prompt. */
+const DEFAULT_MAX_PATTERNS = 10;
+
+export interface RelevantPattern {
+  sourceEntity: string | null;
+  description: string | null;
+  patternSql: string;
+}
+
+/**
+ * Get relevant approved patterns for a question, filtered by keyword overlap
+ * and confidence threshold.
+ *
+ * @param orgId - Organization ID (null for global).
+ * @param question - The user's question to match against.
+ * @param maxPatterns - Maximum patterns to return (default 10).
+ */
+export async function getRelevantPatterns(
+  orgId: string | null,
+  question: string,
+  maxPatterns: number = DEFAULT_MAX_PATTERNS,
+): Promise<RelevantPattern[]> {
+  const threshold = getConfig()?.learn?.confidenceThreshold ?? 0.7;
+  const allPatterns = await getCachedPatterns(orgId);
+
+  // Filter by confidence threshold
+  const aboveThreshold = allPatterns.filter((p) => p.confidence >= threshold);
+  if (aboveThreshold.length === 0) return [];
+
+  // Score by keyword relevance
+  const questionKeywords = extractKeywords(question);
+  if (questionKeywords.size === 0) return [];
+
+  const scored = aboveThreshold
+    .map((p) => ({ pattern: p, score: scorePattern(p, questionKeywords) }))
+    .filter((s) => s.score > 0)
+    .toSorted((a, b) => b.score - a.score || b.pattern.confidence - a.pattern.confidence)
+    .slice(0, maxPatterns);
+
+  return scored.map((s) => ({
+    sourceEntity: s.pattern.source_entity,
+    description: s.pattern.description,
+    patternSql: s.pattern.pattern_sql,
+  }));
+}
+
+/** Sanitize text for safe prompt injection — truncate and strip markdown headings. */
+function sanitizeForPrompt(text: string, maxLen: number): string {
+  let safe = text.replace(/^#{1,6}\s/gm, "").replace(/\n/g, " ");
+  if (safe.length > maxLen) safe = safe.slice(0, maxLen - 3) + "...";
+  return safe;
+}
+
+/**
+ * Build the learned patterns section for the system prompt.
+ * Returns empty string if no relevant patterns found.
+ */
+export async function buildLearnedPatternsSection(
+  orgId: string | null,
+  question: string,
+  maxPatterns?: number,
+): Promise<string> {
+  try {
+    const patterns = await getRelevantPatterns(orgId, question, maxPatterns);
+    if (patterns.length === 0) return "";
+
+    const lines = patterns.map((p) => {
+      const entity = p.sourceEntity ? `[${p.sourceEntity}]` : "[general]";
+      const desc = sanitizeForPrompt(p.description ?? "Query pattern", 200);
+      const sql = sanitizeForPrompt(p.patternSql, 500);
+      return `- ${entity}: ${desc}\n  SQL: ${sql}`;
+    });
+
+    return [
+      "## Previously successful query patterns (organizational knowledge)",
+      "These patterns have been validated by your organization. Use them as reference when writing similar queries, but the semantic layer definitions above take precedence.",
+      "",
+      ...lines,
+    ].join("\n");
+  } catch (err) {
+    log.warn(
+      { orgId, err: err instanceof Error ? err.message : String(err) },
+      "Failed to build learned patterns section — continuing without patterns",
+    );
+    return "";
+  }
+}

package/templates/docker/src/lib/learn/pattern-proposer.ts

@@ -0,0 +1,87 @@
+/**
+ * Fire-and-forget pattern proposal after successful SQL execution.
+ *
+ * Analyzes executed queries for novelty and proposes them as learned
+ * patterns when they don't match existing YAML query_patterns or
+ * learned_patterns rows.
+ */
+
+import { createLogger, getRequestContext } from "@atlas/api/lib/logger";
+import { hasInternalDB, findPatternBySQL, insertLearnedPattern, incrementPatternCount } from "@atlas/api/lib/db/internal";
+import { normalizeSQL, fingerprintSQL, extractPatternInfo, getYamlPatterns } from "@atlas/api/lib/learn/pattern-analyzer";
+
+const log = createLogger("pattern-proposer");
+
+export interface PatternProposalInput {
+  sql: string;
+  dialect: string;
+  /** Used for debug logging only; not stored in the learned pattern record. */
+  connectionId: string;
+}
+
+/**
+ * Analyze a successfully-executed query and propose it as a learned pattern if novel.
+ * Fire-and-forget: errors are logged but never thrown.
+ */
+export function proposePatternIfNovel(input: PatternProposalInput): void {
+  if (!hasInternalDB()) return;
+
+  void _analyzeAndPropose(input).catch((err) => {
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err), sql: input.sql.slice(0, 200) },
+      "Pattern proposal analysis failed — learned patterns may not be recording",
+    );
+  });
+}
+
+/**
+ * Core analysis logic. Exported for testing.
+ * @internal
+ */
+export async function _analyzeAndPropose(input: PatternProposalInput): Promise<void> {
+  const { sql, dialect, connectionId } = input;
+
+  // 1. Normalize SQL for dedup
+  const normalized = normalizeSQL(sql);
+  if (!normalized || normalized.length < 10) return;
+
+  // 2. Check against YAML query_patterns
+  const yamlPatterns = getYamlPatterns();
+  if (yamlPatterns.has(normalized)) {
+    log.debug({ sql: sql.slice(0, 100) }, "Query matches existing YAML pattern — skipping proposal");
+    return;
+  }
+
+  // 3. Check against learned_patterns table
+  const reqCtx = getRequestContext();
+  const orgId = reqCtx?.user?.activeOrganizationId;
+  const fingerprint = fingerprintSQL(normalized);
+
+  const existing = await findPatternBySQL(orgId, normalized);
+  if (existing) {
+    // Duplicate — bump count and confidence, append source fingerprint
+    incrementPatternCount(existing.id, fingerprint);
+    log.debug(
+      { patternId: existing.id, newCount: existing.repetitionCount + 1 },
+      "Incremented repetition count for existing learned pattern",
+    );
+    return;
+  }
+
+  // 4. Novel pattern — extract metadata and insert
+  const info = extractPatternInfo(sql, dialect);
+
+  insertLearnedPattern({
+    orgId,
+    patternSql: normalized,
+    description: info?.description ?? "Query pattern",
+    sourceEntity: info?.primaryTable ?? "unknown",
+    sourceQueries: [fingerprint],
+    proposedBy: "agent",
+  });
+
+  log.debug(
+    { primaryTable: info?.primaryTable, fingerprint, connectionId },
+    "Proposed novel learned pattern",
+  );
+}

package/templates/docker/src/lib/learn/suggestion-helpers.ts

@@ -0,0 +1,34 @@
+/**
+ * Shared helper for converting QuerySuggestionRow (DB shape) to
+ * QuerySuggestion (wire/camelCase shape). Used by both user-facing
+ * and admin suggestion routes.
+ */
+
+import type { QuerySuggestion } from "@useatlas/types";
+import type { QuerySuggestionRow } from "@atlas/api/lib/db/internal";
+
+export function toQuerySuggestion(row: QuerySuggestionRow): QuerySuggestion {
+  let tablesInvolved: string[] = [];
+  try {
+    tablesInvolved = typeof row.tables_involved === "string"
+      ? JSON.parse(row.tables_involved)
+      : row.tables_involved;
+  } catch {
+    // intentionally ignored: malformed JSONB
+  }
+  return {
+    id: row.id,
+    orgId: row.org_id,
+    description: row.description,
+    patternSql: row.pattern_sql,
+    normalizedHash: row.normalized_hash,
+    tablesInvolved,
+    primaryTable: row.primary_table,
+    frequency: row.frequency,
+    clickedCount: row.clicked_count,
+    score: row.score,
+    lastSeenAt: row.last_seen_at,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+  };
+}

package/templates/docker/src/lib/learn/suggestions.ts

@@ -0,0 +1,139 @@
+/**
+ * Query suggestion analysis engine.
+ *
+ * Reads the audit log, groups queries by normalized SQL fingerprint,
+ * scores by frequency + recency, and upserts into query_suggestions.
+ * Idempotent via INSERT … ON CONFLICT DO UPDATE.
+ */
+
+import { normalizeSQL, fingerprintSQL, extractPatternInfo, getYamlPatterns } from "@atlas/api/lib/learn/pattern-analyzer";
+import { getAuditLogQueries, upsertSuggestion } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+
+const log = createLogger("suggestions");
+
+const MIN_PATTERN_COUNT = 2;
+
+// ── Scoring ────────────────────────────────────────────────────────
+
+/** Score a suggestion by frequency weighted by recency (30-day half-life). */
+export function scoreSuggestion(frequency: number, lastSeenAt: Date): number {
+  const daysSince = (Date.now() - lastSeenAt.getTime()) / (1000 * 60 * 60 * 24);
+  return frequency * (1 / (1 + daysSince / 30));
+}
+
+// ── Grouping ───────────────────────────────────────────────────────
+
+interface AuditRow {
+  sql: string;
+  tables_accessed: string | null;
+  timestamp: string;
+}
+
+export interface GroupedPattern {
+  normalizedSql: string;
+  fingerprint: string;
+  count: number;
+  lastSeen: Date;
+  tables: string[];
+  primaryTable: string | null;
+  description: string;
+}
+
+/** Group audit rows by normalized SQL fingerprint. Exported for testing only. */
+export function _groupAuditRows(rows: AuditRow[]): Map<string, GroupedPattern> {
+  const groups = new Map<string, GroupedPattern>();
+
+  for (const row of rows) {
+    if (!row.sql) continue;
+    const normalized = normalizeSQL(row.sql);
+    if (!normalized) continue;
+    const fp = fingerprintSQL(normalized);
+
+    const existing = groups.get(fp);
+    const rowTime = new Date(row.timestamp);
+
+    if (existing) {
+      existing.count++;
+      if (rowTime > existing.lastSeen) {
+        existing.lastSeen = rowTime;
+      }
+    } else {
+      const info = extractPatternInfo(row.sql);
+      let tables: string[] = [];
+      try {
+        tables = row.tables_accessed ? JSON.parse(row.tables_accessed) : [];
+      } catch {
+        // intentionally ignored: malformed tables_accessed JSON
+      }
+      if (tables.length === 0 && info?.tables) {
+        tables = info.tables;
+      }
+
+      groups.set(fp, {
+        normalizedSql: normalized,
+        fingerprint: fp,
+        count: 1,
+        lastSeen: rowTime,
+        tables,
+        primaryTable: info?.primaryTable ?? tables[0] ?? null,
+        description: info?.description ?? "Query pattern",
+      });
+    }
+  }
+
+  return groups;
+}
+
+// ── Batch generation ───────────────────────────────────────────────
+
+/** Batch-generate suggestions from audit log. Idempotent via upsert. */
+export async function generateSuggestions(
+  orgId: string | null
+): Promise<{ created: number; updated: number }> {
+  const rows = await getAuditLogQueries(orgId);
+  if (rows.length === 0) {
+    log.info({ orgId }, "No audit log entries found for suggestion generation");
+    return { created: 0, updated: 0 };
+  }
+
+  const groups = _groupAuditRows(rows);
+
+  // Filter: drop low-frequency patterns
+  const candidates = [...groups.values()].filter((g) => g.count >= MIN_PATTERN_COUNT);
+
+  // Filter: drop patterns that exist in YAML query_patterns
+  let yamlPatterns: Set<string> | null = null;
+  try {
+    yamlPatterns = getYamlPatterns();
+  } catch {
+    // intentionally ignored: YAML patterns unavailable (SaaS mode)
+  }
+
+  const filtered = yamlPatterns
+    ? candidates.filter((c) => !yamlPatterns!.has(c.normalizedSql))
+    : candidates;
+
+  let created = 0;
+  let updated = 0;
+
+  for (const pattern of filtered) {
+    const score = scoreSuggestion(pattern.count, pattern.lastSeen);
+    const result = await upsertSuggestion({
+      orgId,
+      description: pattern.description,
+      patternSql: pattern.normalizedSql,
+      normalizedHash: pattern.fingerprint,
+      tablesInvolved: pattern.tables,
+      primaryTable: pattern.primaryTable,
+      frequency: pattern.count,
+      score,
+      lastSeenAt: pattern.lastSeen,
+    });
+    if (result === "created") created++;
+    else if (result === "updated") updated++;
+  }
+
+  log.info({ orgId, created, updated, total: filtered.length }, "Suggestion generation complete");
+  return { created, updated };
+}

package/templates/docker/src/lib/linear/store.ts

@@ -0,0 +1,200 @@
+/**
+ * Linear installation storage.
+ *
+ * Stores per-workspace Linear API keys in the internal database.
+ * Each workspace admin enters their own API key (BYOT).
+ */
+
+import { hasInternalDB, internalQuery } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+import type { LinearInstallation, LinearInstallationWithSecret } from "@atlas/api/lib/integrations/types";
+
+export type { LinearInstallation, LinearInstallationWithSecret } from "@atlas/api/lib/integrations/types";
+
+const log = createLogger("linear-store");
+
+// ---------------------------------------------------------------------------
+// Shared row parser
+// ---------------------------------------------------------------------------
+
+function parseInstallationRow(
+  row: Record<string, unknown>,
+  context: Record<string, unknown>,
+): LinearInstallationWithSecret | null {
+  const userId = row.user_id;
+  const apiKey = row.api_key;
+  if (typeof userId !== "string" || !userId || typeof apiKey !== "string" || !apiKey) {
+    log.warn(context, "Invalid Linear installation record in database");
+    return null;
+  }
+  return {
+    user_id: userId,
+    api_key: apiKey,
+    user_name: typeof row.user_name === "string" ? row.user_name : null,
+    user_email: typeof row.user_email === "string" ? row.user_email : null,
+    org_id: typeof row.org_id === "string" ? row.org_id : null,
+    installed_at: typeof row.installed_at === "string" ? row.installed_at : new Date().toISOString(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Read operations
+// ---------------------------------------------------------------------------
+
+/**
+ * Get the Linear installation for a user ID (Linear viewer ID).
+ */
+export async function getLinearInstallation(
+  userId: string,
+): Promise<LinearInstallationWithSecret | null> {
+  if (!hasInternalDB()) {
+    return null;
+  }
+
+  try {
+    const rows = await internalQuery<Record<string, unknown>>(
+      "SELECT user_id, api_key, user_name, user_email, org_id, installed_at::text FROM linear_installations WHERE user_id = $1",
+      [userId],
+    );
+    if (rows.length > 0) {
+      return parseInstallationRow(rows[0], { userId });
+    }
+    return null;
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), userId },
+      "Failed to query linear_installations",
+    );
+    throw err;
+  }
+}
+
+/**
+ * Get the Linear installation for an org. Returns null if not found or
+ * if no internal database is configured.
+ */
+export async function getLinearInstallationByOrg(
+  orgId: string,
+): Promise<LinearInstallation | null> {
+  if (!hasInternalDB()) {
+    return null;
+  }
+
+  try {
+    const rows = await internalQuery<Record<string, unknown>>(
+      "SELECT user_id, api_key, user_name, user_email, org_id, installed_at::text FROM linear_installations WHERE org_id = $1",
+      [orgId],
+    );
+    if (rows.length > 0) {
+      const full = parseInstallationRow(rows[0], { orgId });
+      if (!full) return null;
+      const { api_key: _, ...pub } = full;
+      return pub;
+    }
+    return null;
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), orgId },
+      "Failed to query linear_installations by org",
+    );
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Write operations
+// ---------------------------------------------------------------------------
+
+/**
+ * Save or update a Linear installation (API key submission).
+ * Throws if the Linear user is already bound to a different organization (hijack protection).
+ * Throws if the database write fails.
+ */
+export async function saveLinearInstallation(
+  userId: string,
+  opts: { orgId?: string; userName?: string; userEmail?: string; apiKey: string },
+): Promise<void> {
+  if (!hasInternalDB()) {
+    throw new Error("Cannot save Linear installation — no internal database configured");
+  }
+
+  const orgId = opts.orgId ?? null;
+  const userName = opts.userName ?? null;
+  const userEmail = opts.userEmail ?? null;
+
+  try {
+    // Atomic upsert with hijack protection — the WHERE clause rejects rows
+    // bound to a different org in one statement (no TOCTOU race).
+    const rows = await internalQuery<{ user_id: string }>(
+      `INSERT INTO linear_installations (user_id, api_key, user_name, user_email, org_id)
+       VALUES ($1, $2, $3, $4, $5)
+       ON CONFLICT (user_id) DO UPDATE SET
+         api_key = $2,
+         user_name = COALESCE($3, linear_installations.user_name),
+         user_email = COALESCE($4, linear_installations.user_email),
+         org_id = COALESCE($5, linear_installations.org_id),
+         installed_at = now()
+       WHERE linear_installations.org_id IS NULL OR linear_installations.org_id = $5
+       RETURNING user_id`,
+      [userId, opts.apiKey, userName, userEmail, orgId],
+    );
+
+    if (rows.length === 0) {
+      throw new Error(
+        `Linear user ${userId} is already bound to a different organization. ` +
+        `Disconnect the existing installation first.`,
+      );
+    }
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), userId },
+      "Failed to save linear_installations",
+    );
+    throw err;
+  }
+}
+
+/**
+ * Remove a Linear installation by user ID.
+ * Throws if no internal DB or if the query fails.
+ */
+export async function deleteLinearInstallation(userId: string): Promise<void> {
+  if (!hasInternalDB()) {
+    throw new Error("Cannot delete Linear installation — no internal database configured");
+  }
+
+  try {
+    await internalQuery("DELETE FROM linear_installations WHERE user_id = $1", [userId]);
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), userId },
+      "Failed to delete linear_installations",
+    );
+    throw err;
+  }
+}
+
+/**
+ * Remove the Linear installation for an org.
+ * Returns true if a row was deleted, false if no matching row found.
+ * Throws if no internal DB or if the query fails.
+ */
+export async function deleteLinearInstallationByOrg(orgId: string): Promise<boolean> {
+  if (!hasInternalDB()) {
+    throw new Error("Cannot delete Linear installation — no internal database configured");
+  }
+
+  try {
+    const rows = await internalQuery<{ user_id: string }>(
+      "DELETE FROM linear_installations WHERE org_id = $1 RETURNING user_id",
+      [orgId],
+    );
+    return rows.length > 0;
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err.message : String(err), orgId },
+      "Failed to delete linear_installations by org",
+    );
+    throw err;
+  }
+}