@codeyam/codeyam-cli 0.1.0-staging.1669d45 → 0.1.0-staging.27d5a59

package/analyzer-template/packages/database/src/lib/saveFiles.ts

@@ -3,7 +3,7 @@ import dbToFile from './dbToFile';
 import { File } from '~codeyam/types';
 import { Insertable, Selectable } from 'kysely';
 import { FilesTable } from './kysely/tables/filesTable';
-import { randomUUID } from 'crypto';
+const randomUUID = () => crypto.randomUUID();
 
 const BATCH_SIZE = 100;
 

package/analyzer-template/packages/database/src/lib/scenarioToDb.ts

@@ -1,7 +1,7 @@
 import { Scenario } from '~codeyam/types';
 import { type Insertable } from 'kysely';
 import { ScenariosTable } from './kysely/tables/scenariosTable';
-import { randomUUID } from 'crypto';
+const randomUUID = () => crypto.randomUUID();
 
 export default function scenarioToDb(
   scenario: Scenario,

package/analyzer-template/packages/database/src/lib/updateCommitMetadata.ts

@@ -2,16 +2,10 @@
  * updates commit metadata by merging partial updates with existing data.
  * uses database transactions to ensure atomic updates and handle concurrent modifications.
  */
-import {
-  Commit,
-  CommitMetadata,
-  CommitRunStatus,
-  DeepReadonly,
-} from '~codeyam/types';
+import { CommitMetadata, CommitRunStatus } from '~codeyam/types';
 import { getDatabase } from './kysely/db';
 import { awsLog } from '~codeyam/utils';
 import { deepMerge, DeepUpdate } from './jsonUpdateUtils';
-import dbToCommit from './dbToCommit';
 
 type UpdateCommitMetadataArgs = {
   commitId?: string;
@@ -19,10 +13,7 @@ type UpdateCommitMetadataArgs = {
   metadataUpdate?: DeepUpdate<CommitMetadata>;
   runStatusUpdate?: DeepUpdate<CommitRunStatus>;
   archiveCurrentRun?: boolean;
-  updateCallback?: (
-    metadata: CommitMetadata,
-    commit: DeepReadonly<Commit>,
-  ) => void | Promise<void>;
+  updateCallback?: (metadata: CommitMetadata) => void | Promise<void>;
 };
 
 export default async function updateCommitMetadata({
@@ -33,155 +24,173 @@ export default async function updateCommitMetadata({
   archiveCurrentRun,
   updateCallback,
 }: UpdateCommitMetadataArgs): Promise<CommitMetadata | null> {
-  try {
-    return await getDatabase()
-      .transaction()
-      .execute(async (trx) => {
-        const commit = await trx
-          .selectFrom('commits')
-          .selectAll()
-          .$if(!!commitId, (qb: any) => qb.where('id', '=', commitId))
-          .$if(!!commitSha, (qb: any) => qb.where('sha', '=', commitSha))
-          .executeTakeFirst();
-
-        if (!commit) {
-          awsLog(
-            `CodeYam Error: updateCommitMetadata(): Commit ${commitId} not found`,
-          );
-          return null;
-        }
-
-        // Initialize metadata if it doesn't exist (metadata is already parsed from DB)
-        const currentMetadata: CommitMetadata = commit.metadata || {};
-
-        // if the convenience parameter for latest run was supplied, merge it in
-        if (runStatusUpdate) {
-          runStatusUpdate.lastUpdatedAt ??= new Date().toISOString();
-
-          if (runStatusUpdate.currentEntityShas !== undefined) {
+  const MAX_RETRIES = 4;
+  const BASE_DELAY_MS = 250;
+
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await getDatabase()
+        .transaction()
+        .execute(async (trx) => {
+          // Only select what we need - the files column can be 1MB+ and kills performance
+          // If updateCallback is provided, it might need the full commit, so use selectAll
+          const commit = await trx
+            .selectFrom('commits')
+            .select(['id', 'metadata'])
+            .$if(!!commitId, (qb: any) => qb.where('id', '=', commitId))
+            .$if(!!commitSha, (qb: any) => qb.where('sha', '=', commitSha))
+            .executeTakeFirst();
+
+          if (!commit) {
+            awsLog(
+              `CodeYam Error: updateCommitMetadata(): Commit ${commitId} not found`,
+            );
+            return null;
+          }
+
+          // Initialize metadata if it doesn't exist (metadata is already parsed from DB)
+          const currentMetadata: CommitMetadata = commit.metadata || {};
+
+          // if the convenience parameter for latest run was supplied, merge it in
+          if (runStatusUpdate) {
+            runStatusUpdate.lastUpdatedAt ??= new Date().toISOString();
+
+            if (runStatusUpdate.currentEntityShas !== undefined) {
+              console.log(
+                `[updateCommitMetadata] Updating currentRun.currentEntityShas`,
+              );
+              console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
+              console.log(
+                `[updateCommitMetadata] Previous entity SHAs:`,
+                currentMetadata.currentRun?.currentEntityShas,
+              );
+              console.log(
+                `[updateCommitMetadata] New entity SHAs:`,
+                runStatusUpdate.currentEntityShas,
+              );
+              console.log(
+                `[updateCommitMetadata] Archive flag:`,
+                archiveCurrentRun,
+              );
+            }
+
+            metadataUpdate = deepMerge(metadataUpdate ?? {}, {
+              currentRun: runStatusUpdate,
+            });
+          } else if (!metadataUpdate && !updateCallback) {
+            return currentMetadata; // nothing to update
+          }
+
+          // Deep merge the baseline status updates
+          const updatedMetadata: CommitMetadata = metadataUpdate
+            ? deepMerge(currentMetadata, metadataUpdate)
+            : currentMetadata;
+
+          // Archive current run if requested
+          if (archiveCurrentRun && updatedMetadata.currentRun) {
             console.log(
-              `[updateCommitMetadata] Updating currentRun.currentEntityShas`,
+              `[updateCommitMetadata] ========================================`,
             );
+            console.log(`[updateCommitMetadata] ARCHIVING CURRENT RUN`);
             console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
             console.log(
-              `[updateCommitMetadata] Previous entity SHAs:`,
-              currentMetadata.currentRun?.currentEntityShas,
+              `[updateCommitMetadata] Current run entity SHAs:`,
+              updatedMetadata.currentRun.currentEntityShas,
+            );
+            console.log(
+              `[updateCommitMetadata] Current run PIDs: analyzer=${updatedMetadata.currentRun.analyzerPid}, capture=${updatedMetadata.currentRun.capturePid}`,
             );
             console.log(
-              `[updateCommitMetadata] New entity SHAs:`,
-              runStatusUpdate.currentEntityShas,
+              `[updateCommitMetadata] Current run completed: analyses=${updatedMetadata.currentRun.analysesCompleted}, captures=${updatedMetadata.currentRun.capturesCompleted}`,
             );
             console.log(
-              `[updateCommitMetadata] Archive flag:`,
-              archiveCurrentRun,
+              `[updateCommitMetadata] Historical runs before archiving: ${updatedMetadata.historicalRuns?.length || 0}`,
+            );
+
+            const runToArchive: CommitRunStatus = {
+              ...updatedMetadata.currentRun,
+              archivedAt: new Date().toISOString(),
+            };
+
+            console.log(
+              `[updateCommitMetadata] Run to archive:`,
+              JSON.stringify(runToArchive, null, 2),
+            );
+
+            updatedMetadata.historicalRuns = [
+              ...(updatedMetadata.historicalRuns || []),
+              runToArchive,
+            ];
+
+            console.log(
+              `[updateCommitMetadata] Historical runs after archiving: ${updatedMetadata.historicalRuns.length}`,
+            );
+            console.log(
+              `[updateCommitMetadata] All historical runs:`,
+              JSON.stringify(
+                updatedMetadata.historicalRuns.map((r) => ({
+                  entityShas: r.currentEntityShas,
+                  archivedAt: r.archivedAt,
+                  completed: {
+                    analyses: r.analysesCompleted,
+                    captures: r.capturesCompleted,
+                  },
+                })),
+                null,
+                2,
+              ),
+            );
+            console.log(
+              `[updateCommitMetadata] ========================================`,
             );
           }
 
-          metadataUpdate = deepMerge(metadataUpdate ?? {}, {
-            currentRun: runStatusUpdate,
-          });
-        } else if (!metadataUpdate && !updateCallback) {
-          return currentMetadata; // nothing to update
-        }
-
-        // Deep merge the baseline status updates
-        const updatedMetadata: CommitMetadata = metadataUpdate
-          ? deepMerge(currentMetadata, metadataUpdate)
-          : currentMetadata;
-
-        // Archive current run if requested
-        if (archiveCurrentRun && updatedMetadata.currentRun) {
-          console.log(
-            `[updateCommitMetadata] ========================================`,
-          );
-          console.log(`[updateCommitMetadata] ARCHIVING CURRENT RUN`);
-          console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
-          console.log(
-            `[updateCommitMetadata] Current run entity SHAs:`,
-            updatedMetadata.currentRun.currentEntityShas,
-          );
-          console.log(
-            `[updateCommitMetadata] Current run PIDs: analyzer=${updatedMetadata.currentRun.analyzerPid}, capture=${updatedMetadata.currentRun.capturePid}`,
-          );
-          console.log(
-            `[updateCommitMetadata] Current run completed: analyses=${updatedMetadata.currentRun.analysesCompleted}, captures=${updatedMetadata.currentRun.capturesCompleted}`,
-          );
-          console.log(
-            `[updateCommitMetadata] Historical runs before archiving: ${updatedMetadata.historicalRuns?.length || 0}`,
-          );
-
-          const runToArchive: CommitRunStatus = {
-            ...updatedMetadata.currentRun,
-            archivedAt: new Date().toISOString(),
-          };
-
-          console.log(
-            `[updateCommitMetadata] Run to archive:`,
-            JSON.stringify(runToArchive, null, 2),
-          );
-
-          updatedMetadata.historicalRuns = [
-            ...(updatedMetadata.historicalRuns || []),
-            runToArchive,
-          ];
-
-          console.log(
-            `[updateCommitMetadata] Historical runs after archiving: ${updatedMetadata.historicalRuns.length}`,
-          );
-          console.log(
-            `[updateCommitMetadata] All historical runs:`,
-            JSON.stringify(
-              updatedMetadata.historicalRuns.map((r) => ({
-                entityShas: r.currentEntityShas,
-                archivedAt: r.archivedAt,
-                completed: {
-                  analyses: r.analysesCompleted,
-                  captures: r.capturesCompleted,
-                },
-              })),
-              null,
-              2,
-            ),
-          );
-          console.log(
-            `[updateCommitMetadata] ========================================`,
-          );
-        }
-
-        // Allow callback to perform complex atomic updates
-        if (updateCallback) {
-          await updateCallback(updatedMetadata, dbToCommit(commit));
-        }
-
-        try {
-          const result = await trx
-            .updateTable('commits')
-            .set({ metadata: JSON.stringify(updatedMetadata) })
-            .where('id', '=', commit.id)
-            .returningAll()
-            .executeTakeFirst();
+          // Allow callback to perform complex atomic updates
+          if (updateCallback) {
+            await updateCallback(updatedMetadata);
+          }
 
-          if (!result) {
+          try {
+            const result = await trx
+              .updateTable('commits')
+              .set({ metadata: JSON.stringify(updatedMetadata) })
+              .where('id', '=', commit.id)
+              .returning(['id']) // Don't return all columns - files can be 1MB+
+              .executeTakeFirst();
+
+            if (!result) {
+              awsLog(
+                `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
+              );
+              return currentMetadata;
+            }
+
+            return updatedMetadata;
+          } catch (error) {
             awsLog(
               `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
+              error,
             );
             return currentMetadata;
           }
-
-          return updatedMetadata;
-        } catch (error) {
-          awsLog(
-            `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
-            error,
-          );
-          return currentMetadata;
-        }
-      });
-  } catch (error) {
-    awsLog(
-      `CodeYam Error: updateCommitMetadata(): Transaction failed for commit ${commitId}`,
-      error,
-    );
-    return null;
+        });
+    } catch (error) {
+      const isBusy =
+        error instanceof Error && error.message.includes('database is locked');
+
+      if (isBusy && attempt < MAX_RETRIES) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+
+      awsLog(
+        `CodeYam Error: updateCommitMetadata(): Transaction failed for commit ${commitId}${isBusy ? ` after ${attempt + 1} attempts` : ''}`,
+        error,
+      );
+      return null;
+    }
   }
+
+  return null;
 }

package/analyzer-template/packages/database/src/lib/updateFreshAnalysisStatus.ts

@@ -12,53 +12,69 @@ export default async function updateFreshAnalysisStatus(
   fn: (status: AnalysisStatus, analysis: ReadonlyAnalysis) => void,
   source = 'capture',
 ): Promise<AnalysisStatus | null> {
-  try {
-    return await getDatabase()
-      .transaction()
-      .execute(async (trx) => {
-        const dbAnalysis = await trx
-          .selectFrom('analyses')
-          .selectAll()
-          .where('id', '=', id)
-          .executeTakeFirst();
+  const MAX_RETRIES = 4;
+  const BASE_DELAY_MS = 250;
 
-        if (!dbAnalysis) {
-          awsLog(
-            `CodeYam Error: updateFreshAnalysisStatus(): Analysis ${id} not found (source: ${source})`,
-            null,
-            { analysisId: id, source },
-          );
-          return null;
-        }
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await getDatabase()
+        .transaction()
+        .execute(async (trx) => {
+          const dbAnalysis = await trx
+            .selectFrom('analyses')
+            .selectAll()
+            .where('id', '=', id)
+            .executeTakeFirst();
 
-        const analysis = dbToAnalysis(dbAnalysis);
+          if (!dbAnalysis) {
+            awsLog(
+              `CodeYam Error: updateFreshAnalysisStatus(): Analysis ${id} not found (source: ${source})`,
+              null,
+              { analysisId: id, source },
+            );
+            return null;
+          }
 
-        fn(analysis.status, analysis);
+          const analysis = dbToAnalysis(dbAnalysis);
 
-        const result = await trx
-          .updateTable('analyses')
-          .set({ status: JSON.stringify(analysis.status) })
-          .where('id', '=', id)
-          .returningAll()
-          .executeTakeFirst();
+          fn(analysis.status, analysis);
 
-        if (!result) {
-          awsLog(
-            `CodeYam Error: updateFreshAnalysisStatus(): Failed to update analysis ${id} (source: ${source})`,
-            null,
-            { analysisId: id, source },
-          );
-          return null;
-        }
+          const result = await trx
+            .updateTable('analyses')
+            .set({ status: JSON.stringify(analysis.status) })
+            .where('id', '=', id)
+            .returningAll()
+            .executeTakeFirst();
 
-        return analysis.status;
-      });
-  } catch (error) {
-    awsLog(
-      `CodeYam Error: updateFreshAnalysisStatus(): Transaction failed for analysis ${id} (source: ${source})`,
-      error,
-      { analysisId: id, source },
-    );
-    return null;
+          if (!result) {
+            awsLog(
+              `CodeYam Error: updateFreshAnalysisStatus(): Failed to update analysis ${id} (source: ${source})`,
+              null,
+              { analysisId: id, source },
+            );
+            return null;
+          }
+
+          return analysis.status;
+        });
+    } catch (error) {
+      const isBusy =
+        error instanceof Error && error.message.includes('database is locked');
+
+      if (isBusy && attempt < MAX_RETRIES) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+
+      awsLog(
+        `CodeYam Error: updateFreshAnalysisStatus(): Transaction failed for analysis ${id} (source: ${source})${isBusy ? ` after ${attempt + 1} attempts` : ''}`,
+        error,
+        { analysisId: id, source },
+      );
+      return null;
+    }
   }
+
+  return null;
 }

package/analyzer-template/packages/database/src/lib/updateFreshAnalysisStatusWithScenarios.ts

@@ -16,79 +16,95 @@ export default async function updateFreshAnalysisStatusWithScenarios(
   ) => void,
   source = 'capture',
 ): Promise<{ status: AnalysisStatus; scenarios: Scenario[] } | null> {
-  try {
-    return await getDatabase()
-      .transaction()
-      .execute(async (trx) => {
-        // Load analysis
-        const dbAnalysis = await trx
-          .selectFrom('analyses')
-          .selectAll()
-          .where('id', '=', id)
-          .executeTakeFirst();
+  const MAX_RETRIES = 4;
+  const BASE_DELAY_MS = 250;
 
-        if (!dbAnalysis) {
-          awsLog(
-            `CodeYam Error: updateFreshAnalysisStatusWithScenarios(): Analysis ${id} not found (source: ${source})`,
-            null,
-            { analysisId: id, source },
-          );
-          return null;
-        }
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await getDatabase()
+        .transaction()
+        .execute(async (trx) => {
+          // Load analysis
+          const dbAnalysis = await trx
+            .selectFrom('analyses')
+            .selectAll()
+            .where('id', '=', id)
+            .executeTakeFirst();
 
-        // Load scenarios for this analysis
-        const dbScenarios = await trx
-          .selectFrom('scenarios')
-          .selectAll()
-          .where('analysis_id', '=', id)
-          .execute();
+          if (!dbAnalysis) {
+            awsLog(
+              `CodeYam Error: updateFreshAnalysisStatusWithScenarios(): Analysis ${id} not found (source: ${source})`,
+              null,
+              { analysisId: id, source },
+            );
+            return null;
+          }
+
+          // Load scenarios for this analysis
+          const dbScenarios = await trx
+            .selectFrom('scenarios')
+            .selectAll()
+            .where('analysis_id', '=', id)
+            .execute();
 
-        const analysis = dbToAnalysis(dbAnalysis);
+          const analysis = dbToAnalysis(dbAnalysis);
 
-        // Convert DB scenarios to Scenario type
-        const scenarios: Scenario[] = dbScenarios.map((s) => ({
-          id: s.id,
-          projectId: s.project_id,
-          analysisId: s.analysis_id,
-          name: s.name,
-          description: s.description,
-          metadata: s.metadata || {},
-          previousVersionId: s.previous_version_id,
-        }));
+          // Convert DB scenarios to Scenario type
+          const scenarios: Scenario[] = dbScenarios.map((s) => ({
+            id: s.id,
+            projectId: s.project_id,
+            analysisId: s.analysis_id,
+            name: s.name,
+            description: s.description,
+            metadata: s.metadata || {},
+            previousVersionId: s.previous_version_id,
+          }));
 
-        fn(analysis.status, scenarios, analysis);
+          fn(analysis.status, scenarios, analysis);
 
-        // Update analysis status
-        await trx
-          .updateTable('analyses')
-          .set({ status: JSON.stringify(analysis.status) })
-          .where('id', '=', id)
-          .execute();
+          // Update analysis status
+          await trx
+            .updateTable('analyses')
+            .set({ status: JSON.stringify(analysis.status) })
+            .where('id', '=', id)
+            .execute();
 
-        // Update scenarios if they were modified
-        for (const scenario of scenarios) {
-          if (scenario.id) {
-            await trx
-              .updateTable('scenarios')
-              .set({
-                name: scenario.name,
-                description: scenario.description,
-                metadata: JSON.stringify(scenario.metadata),
-                previous_version_id: scenario.previousVersionId,
-              })
-              .where('id', '=', scenario.id)
-              .execute();
+          // Update scenarios if they were modified
+          for (const scenario of scenarios) {
+            if (scenario.id) {
+              await trx
+                .updateTable('scenarios')
+                .set({
+                  name: scenario.name,
+                  description: scenario.description,
+                  metadata: JSON.stringify(scenario.metadata),
+                  previous_version_id: scenario.previousVersionId,
+                })
+                .where('id', '=', scenario.id)
+                .execute();
+            }
           }
-        }
 
-        return { status: analysis.status, scenarios };
-      });
-  } catch (error) {
-    awsLog(
-      `CodeYam Error: updateFreshAnalysisStatusWithScenarios(): Transaction failed for analysis ${id} (source: ${source})`,
-      error,
-      { analysisId: id, source },
-    );
-    return null;
+          return { status: analysis.status, scenarios };
+        });
+    } catch (error) {
+      const isBusy =
+        error instanceof Error && error.message.includes('database is locked');
+
+      if (isBusy && attempt < MAX_RETRIES) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+
+      awsLog(
+        `CodeYam Error: updateFreshAnalysisStatusWithScenarios(): Transaction failed for analysis ${id} (source: ${source})${isBusy ? ` after ${attempt + 1} attempts` : ''}`,
+        error,
+        { analysisId: id, source },
+      );
+      return null;
+    }
   }
+
+  return null;
 }

package/analyzer-template/packages/database/src/lib/userScenarioToDb.ts

@@ -1,7 +1,7 @@
 import { UserScenario } from '~codeyam/types';
 import { Insertable } from 'kysely';
 import { UserScenariosTable } from './kysely/tables/userScenariosTable';
-import { randomUUID } from 'crypto';
+const randomUUID = () => crypto.randomUUID();
 
 export default function userScenarioToDb(
   userScenario: UserScenario,