@codeyam/codeyam-cli 0.1.0-staging.c1c8678 → 0.1.0-staging.c276166

package/analyzer-template/packages/analyze/src/lib/files/scenarios/mergeInDependentDataStructure.ts

@@ -52,6 +52,7 @@ const PRIMITIVE_TYPES = new Set([
 
 // Check if a type string represents a primitive type
 // Handles union types like "string | undefined" or "number | null"
+// Also handles string literal unions like "'suggestion' | 'warning' | 'tip'"
 function isPrimitiveType(typeStr: string): boolean {
   if (PRIMITIVE_TYPES.has(typeStr)) {
     return true;
@@ -59,9 +60,15 @@ function isPrimitiveType(typeStr: string): boolean {
   // Check union types - if ALL parts of the union are primitives, it's primitive
   // e.g., "string | undefined" -> ["string", "undefined"] -> both are primitive -> true
   // e.g., "object | null" -> ["object", "null"] -> object is not primitive -> false
+  // e.g., "'suggestion' | 'warning'" -> string literal union -> true
   if (typeStr.includes('|')) {
     const parts = typeStr.split('|').map((p) => p.trim());
-    return parts.every((part) => PRIMITIVE_TYPES.has(part));
+    return parts.every(
+      (part) =>
+        PRIMITIVE_TYPES.has(part) ||
+        // String literal values like 'suggestion', 'warning' are primitives
+        (part.startsWith("'") && part.endsWith("'")),
+    );
   }
   return false;
 }
@@ -110,9 +117,21 @@ function pathPartMatches(pathPart: string, schemaPathPart: string): boolean {
 function bestValueFromOptions(options: Array<string | undefined>) {
   options = options.filter(Boolean) as string[];
 
-  const known = options.find((o) => !o.includes('unknown'));
-  if (known) {
-    return known;
+  const known = options.filter((o) => !o.includes('unknown'));
+  if (known.length > 0) {
+    // Among known values, prefer string literal unions over bare primitives.
+    // e.g., "'draft' | 'inProgress' | 'paused' | 'completed'" is more specific than "string".
+    // This handles cases where a dependency schema has a bare type like "string" but the
+    // child entity's analysis has the actual literal union from TypeScript type resolution.
+    if (known.length > 1 && known.some((o) => PRIMITIVE_TYPES.has(o))) {
+      const literalUnion = known.find(
+        (o) => !PRIMITIVE_TYPES.has(o) && o.includes("'"),
+      );
+      if (literalUnion) {
+        return literalUnion;
+      }
+    }
+    return known[0];
   }
 
   const notUnknown = options.find((o) => o !== 'unknown');

package/analyzer-template/packages/aws/package.json

@@ -11,15 +11,15 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@aws-sdk/client-codebuild": "^3.948.0",
-    "@aws-sdk/client-cloudwatch-logs": "^3.980.0",
-    "@aws-sdk/client-dynamodb": "^3.956.0",
-    "@aws-sdk/client-ecr": "^3.948.0",
-    "@aws-sdk/client-ecs": "^3.956.0",
-    "@aws-sdk/client-s3": "^3.940.0",
-    "@aws-sdk/client-sqs": "^3.956.0",
-    "@aws-sdk/lib-storage": "^3.975.0",
-    "@aws-sdk/s3-request-presigner": "^3.940.0",
-    "@aws-sdk/util-dynamodb": "^3.971.0"
+    "@aws-sdk/client-codebuild": "^3.1000.0",
+    "@aws-sdk/client-cloudwatch-logs": "^3.990.0",
+    "@aws-sdk/client-dynamodb": "^3.990.0",
+    "@aws-sdk/client-ecr": "^3.990.0",
+    "@aws-sdk/client-ecs": "^3.1004.0",
+    "@aws-sdk/client-s3": "^3.990.0",
+    "@aws-sdk/client-sqs": "^3.990.0",
+    "@aws-sdk/lib-storage": "^3.990.0",
+    "@aws-sdk/s3-request-presigner": "^3.990.0",
+    "@aws-sdk/util-dynamodb": "^3.990.0"
   }
 }

package/analyzer-template/packages/database/index.ts

@@ -75,6 +75,7 @@ export * from './src/lib/kysely/tables/backgroundJobsTable';
 export * from './src/lib/kysely/tables/branchesTable';
 export * from './src/lib/kysely/tables/commitBranchesTable';
 export * from './src/lib/kysely/tables/commitsTable';
+export * from './src/lib/kysely/tables/editorScenariosTable';
 export * from './src/lib/kysely/tables/entitiesTable';
 export * from './src/lib/kysely/tables/entityBranchesTable';
 export * from './src/lib/kysely/tables/entityStatementsTable';

package/analyzer-template/packages/database/package.json

@@ -13,10 +13,10 @@
   "license": "ISC",
   "dependencies": {
     "@supabase/supabase-js": "2.88.0",
-    "better-sqlite3": "^12.4.1",
+    "better-sqlite3": "^12.6.2",
     "fetch-retry": "^6.0.0",
     "kysely": "^0.28.11",
-    "pg": "^8.16.3"
+    "pg": "^8.19.0"
   },
   "jest": {
     "preset": "ts-jest",
@@ -25,7 +25,7 @@
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^30.0.0",
-    "@types/pg": "^8.15.5",
+    "@types/pg": "^8.18.0",
     "@types/ws": "^8.18.1"
   }
 }

package/analyzer-template/packages/database/src/lib/kysely/db.ts

@@ -62,6 +62,10 @@ import {
   createScenarioCommentsTable,
   ScenarioCommentsTable,
 } from './tables/scenarioCommentsTable';
+import {
+  createEditorScenariosTable,
+  EditorScenariosTable,
+} from './tables/editorScenariosTable';
 import { createScenariosTable, ScenariosTable } from './tables/scenariosTable';
 import {
   createStatementsTable,
@@ -106,6 +110,7 @@ const databaseSchema = {
   commit_branches: schemaField<CommitBranchesTable>(),
   commits: schemaField<CommitsTable>(),
   debug_reports: schemaField<DebugBundlesTable>(),
+  editor_scenarios: schemaField<EditorScenariosTable>(),
   entities: schemaField<EntitiesTable>(),
   entity_branches: schemaField<EntityBranchesTable>(),
   entity_statements: schemaField<EntityStatementsTable>(),
@@ -210,6 +215,7 @@ export function getSqliteDatabase(sqlitePath?: string) {
     fileMustExist: false,
   });
   sqlite.pragma('journal_mode = WAL');
+  sqlite.pragma('busy_timeout = 5000');
   sqlite.pragma('synchronous = FULL');
 
   // Verify the database has the expected schema
@@ -305,6 +311,7 @@ export async function getDatabaseForTesting(
     // Standard SQLite testing path
     const sqlite = new SQLiteDatabase(dbPath);
     sqlite.pragma('journal_mode = WAL');
+    sqlite.pragma('busy_timeout = 5000');
     sqlite.pragma('synchronous = FULL');
     const kyselyDb = new Kysely<Database>({
       dialect: new SqliteDialect({ database: sqlite }),
@@ -390,6 +397,7 @@ export async function createSqliteSchema(db: Kysely<Database>): Promise<void> {
   await createCommitBranchesTable(db);
   await createCommitsTable(db);
   await createDebugBundlesTable(db);
+  await createEditorScenariosTable(db);
   await createEntitiesTable(db);
   await createEntityBranchesTable(db);
   await createEntityStatementsTable(db);

package/analyzer-template/packages/database/src/lib/kysely/tables/editorScenariosTable.ts

@@ -0,0 +1,138 @@
+import { Kysely, ColumnDefinitionBuilder } from 'kysely';
+import type { Generated } from 'kysely';
+
+import { schemaField, defaultNow } from '../schemaHelpers';
+
+const editorScenariosSchema = {
+  id: schemaField<Generated<string>>(),
+  project_id: schemaField<string>(),
+  name: schemaField<string>(),
+  description: schemaField<string | null>(),
+  component_name: schemaField<string | null>(),
+  component_path: schemaField<string | null>(),
+  url: schemaField<string | null>(),
+  type: schemaField<string | null>(),
+  screenshot_path: schemaField<string | null>(),
+  viewport_width: schemaField<number | null>(),
+  viewport_height: schemaField<number | null>(),
+  dimension: schemaField<string | null>(),
+  dimensions: schemaField<string | null>(),
+  screenshot_paths: schemaField<string | null>(),
+  created_at: schemaField<Generated<string>>(),
+  updated_at: schemaField<Generated<string>>(),
+} as const;
+
+export type EditorScenariosTable = {
+  [K in keyof typeof editorScenariosSchema]: (typeof editorScenariosSchema)[K];
+};
+
+export const EditorScenariosTableColumns = Object.keys(
+  editorScenariosSchema,
+) as (keyof EditorScenariosTable)[];
+
+export async function createEditorScenariosTable(
+  db: Kysely<any>,
+): Promise<void> {
+  await db.schema
+    .createTable('editor_scenarios')
+    .addColumn('id', 'uuid', (col: ColumnDefinitionBuilder) => col.primaryKey())
+    .addColumn('project_id', 'uuid', (col: ColumnDefinitionBuilder) =>
+      col.notNull(),
+    )
+    .addColumn('name', 'varchar', (col: ColumnDefinitionBuilder) =>
+      col.notNull(),
+    )
+    .addColumn('description', 'text')
+    .addColumn('component_name', 'varchar')
+    .addColumn('component_path', 'varchar')
+    .addColumn('url', 'varchar')
+    .addColumn('type', 'varchar')
+    .addColumn('screenshot_path', 'varchar')
+    .addColumn('created_at', 'datetime', defaultNow(true))
+    .addColumn('updated_at', 'datetime', defaultNow(true))
+    .ifNotExists()
+    .execute();
+
+  // Add columns to existing tables (ALTER TABLE has no IF NOT EXISTS in SQLite)
+  for (const col of [
+    'component_name',
+    'component_path',
+    'url',
+    'type',
+    'screenshot_path',
+  ] as const) {
+    try {
+      await db.schema
+        .alterTable('editor_scenarios')
+        .addColumn(col, 'varchar')
+        .execute();
+    } catch {
+      // Column already exists — expected for fresh databases
+    }
+  }
+
+  // Integer columns for viewport dimensions
+  for (const col of ['viewport_width', 'viewport_height'] as const) {
+    try {
+      await db.schema
+        .alterTable('editor_scenarios')
+        .addColumn(col, 'integer')
+        .execute();
+    } catch {
+      // Column already exists — expected for fresh databases
+    }
+  }
+
+  // Named dimension column (e.g. "Mobile", "Desktop")
+  try {
+    await db.schema
+      .alterTable('editor_scenarios')
+      .addColumn('dimension', 'varchar')
+      .execute();
+  } catch {
+    // Column already exists — expected for fresh databases
+  }
+
+  // Multi-dimension support: JSON array of dimension names and map of screenshot paths
+  for (const col of ['dimensions', 'screenshot_paths'] as const) {
+    try {
+      await db.schema
+        .alterTable('editor_scenarios')
+        .addColumn(col, 'text')
+        .execute();
+    } catch {
+      // Column already exists — expected for fresh databases
+    }
+  }
+
+  // Backfill: wrap existing single dimension into dimensions array,
+  // and existing screenshot_path into screenshot_paths object
+  try {
+    const rows = await db
+      .selectFrom('editor_scenarios')
+      .select(['id', 'dimension', 'screenshot_path'])
+      .execute();
+    for (const row of rows) {
+      const r = row as any;
+      if (r.dimension && !r.dimensions) {
+        await db
+          .updateTable('editor_scenarios')
+          .set({ dimensions: JSON.stringify([r.dimension]) })
+          .where('id', '=', r.id)
+          .execute();
+      }
+      if (r.screenshot_path && !r.screenshot_paths) {
+        const dimName = r.dimension || 'Default';
+        await db
+          .updateTable('editor_scenarios')
+          .set({
+            screenshot_paths: JSON.stringify({ [dimName]: r.screenshot_path }),
+          })
+          .where('id', '=', r.id)
+          .execute();
+      }
+    }
+  } catch {
+    // Backfill is best-effort — columns may not exist yet on first run
+  }
+}

package/analyzer-template/packages/database/src/lib/loadCommits.ts

@@ -9,6 +9,7 @@ import {
 import { awsLog } from '~codeyam/utils';
 import { Commit } from '~codeyam/types';
 import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+import { CommitsTableColumnsLite } from './kysely/tables/commitsTable';
 import { sql, ExpressionBuilder } from 'kysely';
 
 interface LoadCommitsArgs {
@@ -150,25 +151,35 @@ export default async function loadCommits({
   const startTime = Date.now();
 
   try {
-    let query = db
-      .selectFrom('commits')
-      .selectAll('commits')
-      .select((eb: ExpressionBuilder<any, any>) => [
-        jsonObjectFrom(
-          eb
-            .selectFrom('github_users')
-            .select([
-              'username',
-              'preferred_username as preferredUsername',
-              'avatar_url as avatarUrl',
-            ])
-            .where(
-              'github_users.username',
-              '=',
-              eb.ref('commits.author_github_username'),
-            ),
-        ).as('author'),
-      ]);
+    // When skipRelations is true, use lite columns (excludes 'files' which can be 1MB+ per commit)
+    // and skip the author subquery for maximum efficiency
+    let query: any;
+    if (skipRelations) {
+      const liteColumns = CommitsTableColumnsLite.map(
+        (col) => `commits.${col}` as any,
+      );
+      query = db.selectFrom('commits').select(liteColumns);
+    } else {
+      query = db
+        .selectFrom('commits')
+        .selectAll('commits')
+        .select((eb: ExpressionBuilder<any, any>) => [
+          jsonObjectFrom(
+            eb
+              .selectFrom('github_users')
+              .select([
+                'username',
+                'preferred_username as preferredUsername',
+                'avatar_url as avatarUrl',
+              ])
+              .where(
+                'github_users.username',
+                '=',
+                eb.ref('commits.author_github_username'),
+              ),
+          ).as('author'),
+        ]);
+    }
 
     if (projectId) {
       query = query.where('project_id', '=', projectId);
@@ -215,7 +226,7 @@ export default async function loadCommits({
 
     // DEBUG: Log slow queries (includes files column which can be 1MB+)
     if (queryTime > 100) {
-      const totalFilesSize = commits.reduce((sum, c: any) => {
+      const totalFilesSize = commits.reduce((sum: number, c: any) => {
         return sum + (c.files ? JSON.stringify(c.files).length : 0);
       }, 0);
       console.log(

package/analyzer-template/packages/database/src/lib/loadEntities.ts

@@ -95,12 +95,6 @@ export default async function loadEntities({
     const entities = await query.execute();
 
     if (!entities || entities.length === 0) {
-      console.log('Load Entities: No entities found', {
-        projectId,
-        fileIds,
-        filePaths,
-        shas,
-      });
       return null;
     }
 

package/analyzer-template/packages/database/src/lib/loadReadyToBeCapturedAnalyses.ts

@@ -48,11 +48,6 @@ export default async function loadReadyToBeCapturedAnalyses(
       sql<string>`${sql.ref('analyses.status')} -> 'readyToBeCaptured'`,
       '=',
       'true',
-    )
-    // Exclude entities that are not exported - they cannot be captured because they can't be imported
-    // Use -> operator for cross-database boolean comparison
-    .where(
-      sql<boolean>`(${sql.ref('entities.metadata')} -> 'notExported') IS NULL OR (${sql.ref('entities.metadata')} -> 'notExported') != 'true'`,
     );
 
   if (projectId) {

package/analyzer-template/packages/database/src/lib/updateCommitMetadata.ts

@@ -24,157 +24,108 @@ export default async function updateCommitMetadata({
   archiveCurrentRun,
   updateCallback,
 }: UpdateCommitMetadataArgs): Promise<CommitMetadata | null> {
-  try {
-    return await getDatabase()
-      .transaction()
-      .execute(async (trx) => {
-        // Only select what we need - the files column can be 1MB+ and kills performance
-        // If updateCallback is provided, it might need the full commit, so use selectAll
-        const commit = await trx
-          .selectFrom('commits')
-          .select(['id', 'metadata'])
-          .$if(!!commitId, (qb: any) => qb.where('id', '=', commitId))
-          .$if(!!commitSha, (qb: any) => qb.where('sha', '=', commitSha))
-          .executeTakeFirst();
-
-        if (!commit) {
-          awsLog(
-            `CodeYam Error: updateCommitMetadata(): Commit ${commitId} not found`,
-          );
-          return null;
-        }
-
-        // Initialize metadata if it doesn't exist (metadata is already parsed from DB)
-        const currentMetadata: CommitMetadata = commit.metadata || {};
-
-        // if the convenience parameter for latest run was supplied, merge it in
-        if (runStatusUpdate) {
-          runStatusUpdate.lastUpdatedAt ??= new Date().toISOString();
-
-          if (runStatusUpdate.currentEntityShas !== undefined) {
-            console.log(
-              `[updateCommitMetadata] Updating currentRun.currentEntityShas`,
-            );
-            console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
-            console.log(
-              `[updateCommitMetadata] Previous entity SHAs:`,
-              currentMetadata.currentRun?.currentEntityShas,
-            );
-            console.log(
-              `[updateCommitMetadata] New entity SHAs:`,
-              runStatusUpdate.currentEntityShas,
-            );
-            console.log(
-              `[updateCommitMetadata] Archive flag:`,
-              archiveCurrentRun,
+  const MAX_RETRIES = 4;
+  const BASE_DELAY_MS = 250;
+
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await getDatabase()
+        .transaction()
+        .execute(async (trx) => {
+          // Only select what we need - the files column can be 1MB+ and kills performance
+          // If updateCallback is provided, it might need the full commit, so use selectAll
+          const commit = await trx
+            .selectFrom('commits')
+            .select(['id', 'metadata'])
+            .$if(!!commitId, (qb: any) => qb.where('id', '=', commitId))
+            .$if(!!commitSha, (qb: any) => qb.where('sha', '=', commitSha))
+            .executeTakeFirst();
+
+          if (!commit) {
+            awsLog(
+              `CodeYam Error: updateCommitMetadata(): Commit ${commitId} not found`,
             );
+            return null;
           }
 
-          metadataUpdate = deepMerge(metadataUpdate ?? {}, {
-            currentRun: runStatusUpdate,
-          });
-        } else if (!metadataUpdate && !updateCallback) {
-          return currentMetadata; // nothing to update
-        }
-
-        // Deep merge the baseline status updates
-        const updatedMetadata: CommitMetadata = metadataUpdate
-          ? deepMerge(currentMetadata, metadataUpdate)
-          : currentMetadata;
-
-        // Archive current run if requested
-        if (archiveCurrentRun && updatedMetadata.currentRun) {
-          console.log(
-            `[updateCommitMetadata] ========================================`,
-          );
-          console.log(`[updateCommitMetadata] ARCHIVING CURRENT RUN`);
-          console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
-          console.log(
-            `[updateCommitMetadata] Current run entity SHAs:`,
-            updatedMetadata.currentRun.currentEntityShas,
-          );
-          console.log(
-            `[updateCommitMetadata] Current run PIDs: analyzer=${updatedMetadata.currentRun.analyzerPid}, capture=${updatedMetadata.currentRun.capturePid}`,
-          );
-          console.log(
-            `[updateCommitMetadata] Current run completed: analyses=${updatedMetadata.currentRun.analysesCompleted}, captures=${updatedMetadata.currentRun.capturesCompleted}`,
-          );
-          console.log(
-            `[updateCommitMetadata] Historical runs before archiving: ${updatedMetadata.historicalRuns?.length || 0}`,
-          );
-
-          const runToArchive: CommitRunStatus = {
-            ...updatedMetadata.currentRun,
-            archivedAt: new Date().toISOString(),
-          };
-
-          console.log(
-            `[updateCommitMetadata] Run to archive:`,
-            JSON.stringify(runToArchive, null, 2),
-          );
-
-          updatedMetadata.historicalRuns = [
-            ...(updatedMetadata.historicalRuns || []),
-            runToArchive,
-          ];
-
-          console.log(
-            `[updateCommitMetadata] Historical runs after archiving: ${updatedMetadata.historicalRuns.length}`,
-          );
-          console.log(
-            `[updateCommitMetadata] All historical runs:`,
-            JSON.stringify(
-              updatedMetadata.historicalRuns.map((r) => ({
-                entityShas: r.currentEntityShas,
-                archivedAt: r.archivedAt,
-                completed: {
-                  analyses: r.analysesCompleted,
-                  captures: r.capturesCompleted,
-                },
-              })),
-              null,
-              2,
-            ),
-          );
-          console.log(
-            `[updateCommitMetadata] ========================================`,
-          );
-        }
-
-        // Allow callback to perform complex atomic updates
-        if (updateCallback) {
-          await updateCallback(updatedMetadata);
-        }
-
-        try {
-          const result = await trx
-            .updateTable('commits')
-            .set({ metadata: JSON.stringify(updatedMetadata) })
-            .where('id', '=', commit.id)
-            .returning(['id']) // Don't return all columns - files can be 1MB+
-            .executeTakeFirst();
+          // Initialize metadata if it doesn't exist (metadata is already parsed from DB)
+          const currentMetadata: CommitMetadata = commit.metadata || {};
+
+          // if the convenience parameter for latest run was supplied, merge it in
+          if (runStatusUpdate) {
+            runStatusUpdate.lastUpdatedAt ??= new Date().toISOString();
 
-          if (!result) {
+            metadataUpdate = deepMerge(metadataUpdate ?? {}, {
+              currentRun: runStatusUpdate,
+            });
+          } else if (!metadataUpdate && !updateCallback) {
+            return currentMetadata; // nothing to update
+          }
+
+          // Deep merge the baseline status updates
+          const updatedMetadata: CommitMetadata = metadataUpdate
+            ? deepMerge(currentMetadata, metadataUpdate)
+            : currentMetadata;
+
+          // Archive current run if requested
+          if (archiveCurrentRun && updatedMetadata.currentRun) {
+            const runToArchive: CommitRunStatus = {
+              ...updatedMetadata.currentRun,
+              archivedAt: new Date().toISOString(),
+            };
+
+            updatedMetadata.historicalRuns = [
+              ...(updatedMetadata.historicalRuns || []),
+              runToArchive,
+            ];
+          }
+
+          // Allow callback to perform complex atomic updates
+          if (updateCallback) {
+            await updateCallback(updatedMetadata);
+          }
+
+          try {
+            const result = await trx
+              .updateTable('commits')
+              .set({ metadata: JSON.stringify(updatedMetadata) })
+              .where('id', '=', commit.id)
+              .returning(['id']) // Don't return all columns - files can be 1MB+
+              .executeTakeFirst();
+
+            if (!result) {
+              awsLog(
+                `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
+              );
+              return currentMetadata;
+            }
+
+            return updatedMetadata;
+          } catch (error) {
             awsLog(
               `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
+              error,
             );
             return currentMetadata;
           }
-
-          return updatedMetadata;
-        } catch (error) {
-          awsLog(
-            `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
-            error,
-          );
-          return currentMetadata;
-        }
-      });
-  } catch (error) {
-    awsLog(
-      `CodeYam Error: updateCommitMetadata(): Transaction failed for commit ${commitId}`,
-      error,
-    );
-    return null;
+        });
+    } catch (error) {
+      const isBusy =
+        error instanceof Error && error.message.includes('database is locked');
+
+      if (isBusy && attempt < MAX_RETRIES) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+
+      awsLog(
+        `CodeYam Error: updateCommitMetadata(): Transaction failed for commit ${commitId}${isBusy ? ` after ${attempt + 1} attempts` : ''}`,
+        error,
+      );
+      return null;
+    }
   }
+
+  return null;
 }