@codeyam/codeyam-cli 0.1.0-staging.28f73cf → 0.1.0-staging.596f0eb

package/analyzer-template/packages/database/src/lib/loadEntity.ts

@@ -0,0 +1,105 @@
+import { ExpressionBuilder } from 'kysely';
+import { getDatabase } from './kysely/db';
+import dbToEntity from './dbToEntity';
+import { awsLog } from '~codeyam/utils';
+import { getEntityBranchesAsJson } from './loadEntityBranches';
+
+export interface loadEntityArgs {
+  projectId: string;
+  sha: string;
+}
+
+export interface loadEntityByNameAndPathArgs {
+  projectId: string;
+  name: string;
+  filePath: string;
+}
+
+export default async function loadEntity({ projectId, sha }: loadEntityArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.sha', '=', sha)
+      .executeTakeFirst();
+
+    if (!entity) {
+      // Only log at debug level - Entity not found is expected in E2E baseline mode
+      // where entities are created fresh and don't exist in the database yet
+      if (process.env.CODEYAM_E2E_BASELINE_MODE !== 'true') {
+        awsLog('CodeYam Error: Load Entity: Entity not found', null, {
+          projectId,
+          sha,
+        });
+      }
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      sha,
+    });
+    return null;
+  }
+}
+
+/**
+ * Load entity by name and file path instead of SHA.
+ * This is useful for E2E tests where we want to avoid hardcoding SHAs.
+ */
+export async function loadEntityByNameAndPath({
+  projectId,
+  name,
+  filePath,
+}: loadEntityByNameAndPathArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.name', '=', name)
+      .where('files.path', '=', filePath)
+      .executeTakeFirst();
+
+    if (!entity) {
+      awsLog(
+        'CodeYam Error: Load Entity: Entity not found by name and path',
+        null,
+        {
+          projectId,
+          name,
+          filePath,
+        },
+      );
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      name,
+      filePath,
+    });
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFile.ts

@@ -0,0 +1,42 @@
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+
+export default async function loadFile({
+  projectId,
+  id,
+  path,
+}: {
+  projectId?: string;
+  id?: string;
+  path?: string;
+}) {
+  const db = getDatabase();
+
+  let query = db.selectFrom('files').selectAll();
+
+  if (projectId) {
+    query = query.where('project_id', '=', projectId);
+  }
+
+  if (id) {
+    query = query.where('id', '=', id);
+  }
+
+  if (path) {
+    query = query.where('path', '=', path);
+  }
+
+  try {
+    const file = await query.executeTakeFirst();
+
+    if (!file) {
+      console.log('Error loading file', { projectId, id, path }, 'Not found');
+      return null;
+    }
+
+    return dbToFile(file);
+  } catch (error) {
+    console.log('Error loading file', { projectId, id, path }, error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFiles.ts

@@ -0,0 +1,134 @@
+import { File } from '~codeyam/types';
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+import { Selectable } from 'kysely';
+import { FilesTable } from './kysely/tables/filesTable';
+
+const RANGE_LIMIT = 1000;
+
+/**
+ * Load only file paths for a project (lightweight version)
+ * Use this when you only need paths, not full file objects (~80% memory savings)
+ */
+export async function loadFilePaths(projectId: string): Promise<string[]> {
+  const db = getDatabase();
+  const paths: string[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      const query = db
+        .selectFrom('files')
+        .select('path') // Only select path column
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      paths.push(...filesData.map((f) => f.path));
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return paths;
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading file paths in loadFilePaths',
+      error,
+    );
+    return [];
+  }
+}
+
+interface LoadFilesArgs {
+  projectId: string;
+  filePaths?: string[];
+  fileIds?: string[];
+  fileNames?: string[];
+}
+
+export default async function loadFiles({
+  projectId,
+  filePaths,
+  fileIds,
+  fileNames,
+}: LoadFilesArgs): Promise<File[] | null> {
+  if (filePaths && filePaths.length > 50) {
+    const allFiles: File[] = [];
+    for (let i = 0; i < filePaths.length; i += 50) {
+      const chunk = filePaths.slice(i, i + 50);
+      const chunkFiles = await loadFiles({
+        projectId,
+        filePaths: chunk,
+        fileIds,
+        fileNames,
+      });
+
+      if (chunkFiles) {
+        allFiles.push(...chunkFiles);
+      }
+    }
+    return allFiles;
+  }
+
+  const db = getDatabase();
+  const files: Selectable<FilesTable>[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      let query = db
+        .selectFrom('files')
+        .selectAll()
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      if (filePaths) {
+        if (filePaths.length === 0) return [];
+        query = query.where('path', 'in', filePaths);
+      }
+
+      if (fileIds) {
+        if (fileIds.length === 0) return [];
+        query = query.where('id', 'in', fileIds);
+      }
+
+      if (fileNames) {
+        if (fileNames.length === 0) return [];
+        query = query.where('name', 'in', fileNames);
+      }
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      files.push(...filesData);
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return files?.map(dbToFile);
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading project files in loadFiles',
+      error,
+    );
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadReadyToBeCapturedAnalyses.ts

@@ -0,0 +1,97 @@
+import { Analysis } from '~codeyam/types';
+import dbToAnalysis from './dbToAnalysis';
+import { getDatabase, getJsonHelper } from './kysely/db';
+import { sql, ExpressionBuilder } from 'kysely';
+import { aliasedTableColumns } from './kysely/aggregationHelpers';
+import { EntitiesTableColumns } from './kysely/tables/entitiesTable';
+import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+import { DbAnalysis } from './kysely/tableRelations';
+
+export interface LoadReadyToBeCapturedAnalysesArgs {
+  projectId?: string;
+  ids?: string[];
+  branchCommitSha?: string;
+}
+
+export default async function loadReadyToBeCapturedAnalyses(
+  params: LoadReadyToBeCapturedAnalysesArgs,
+): Promise<Analysis[]> {
+  const { projectId, branchCommitSha, ids } = params;
+
+  const db = getDatabase();
+  const { jsonObjectFrom, jsonArrayFrom } = getJsonHelper();
+
+  let query = db
+    .selectFrom('analyses')
+    .innerJoin('entities', 'entities.sha', 'analyses.entity_sha')
+    .selectAll('analyses')
+    .select((eb: ExpressionBuilder<any, any>) => [
+      jsonObjectFrom(
+        eb
+          .selectFrom('entities')
+          .select(aliasedTableColumns('entities', EntitiesTableColumns))
+          .whereRef('entities.sha', '=', 'analyses.entity_sha')
+          .limit(1),
+      ).as('entity'),
+      jsonArrayFrom(
+        eb
+          .selectFrom('scenarios')
+          .select(aliasedTableColumns('scenarios', ScenariosTableColumns))
+          .whereRef('scenarios.analysis_id', '=', 'analyses.id'),
+      ).as('scenarios'),
+    ])
+    .where(
+      // Use ->> to extract as text for proper comparison (-> returns JSONB which doesn't compare to string)
+      sql<string>` ${sql.ref('analyses.status')} ->> 'readyToBeCaptured'`,
+      '=',
+      'true',
+    )
+    // Exclude entities that are not exported - they cannot be captured because they can't be imported
+    .where(
+      sql<boolean>`(${sql.ref('entities.metadata')} ->> 'notExported') IS NULL OR (${sql.ref('entities.metadata')} ->> 'notExported') != 'true'`,
+    );
+
+  if (projectId) {
+    query = query.where('analyses.project_id', '=', projectId);
+  }
+
+  // Only filter by branchCommitSha if specific IDs are NOT provided
+  // When specific analysis IDs are requested (e.g., debug/recapture), we want those exact analyses
+  // regardless of their branch_commit_sha (which may be NULL in CLI scenarios)
+  if (branchCommitSha && (!ids || ids.length === 0)) {
+    query = query.where('analyses.branch_commit_sha', '=', branchCommitSha);
+  }
+
+  if (ids && ids.length > 0) {
+    query = query.where('analyses.id', 'in', ids);
+  }
+
+  const results = await query.execute();
+
+  if (!results || results.length === 0) {
+    console.log('CodeYam: No analyses found for readyToBeCaptured', {
+      projectId,
+      branchCommitSha,
+      ids: ids?.length,
+    });
+    return [];
+  }
+
+  const analyses = results as DbAnalysis[];
+  const mappedAnalyses = analyses.map(dbToAnalysis);
+
+  // Log entity export status for debugging notExported filtering
+  console.log(
+    'CodeYam: loadReadyToBeCapturedAnalyses results (notExported entities should be filtered)',
+    {
+      count: mappedAnalyses.length,
+      entities: mappedAnalyses.map((a) => ({
+        name: a.entityName,
+        filePath: a.filePath,
+        notExported: a.entity?.metadata?.notExported ?? false,
+      })),
+    },
+  );
+
+  return mappedAnalyses;
+}

package/analyzer-template/packages/database/src/lib/loadStatement.ts

@@ -0,0 +1,23 @@
+import { getDatabase } from './kysely/db';
+import { DbStatement } from '~codeyam/database';
+
+export default async function loadStatement(
+  sha: string,
+): Promise<Pick<DbStatement, 'sha' | 'results'> | null> {
+  try {
+    const result = await getDatabase()
+      .selectFrom('statements')
+      .select(['sha', 'results'])
+      .where('sha', '=', sha)
+      .executeTakeFirst();
+    // for unknown reasons, SQLite parses the JSON in 'results' even though
+    // the field is defined as text. We need to stringify it back.
+    if (result && result.results && typeof result.results !== 'string') {
+      result.results = JSON.stringify(result.results);
+    }
+    return result ?? null;
+  } catch (error) {
+    console.log('Error loading statement', error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/saveEntityStatements.ts

@@ -0,0 +1,27 @@
+import { Entity } from '~codeyam/types';
+import { getDatabase } from './kysely/db';
+import { EntityStatementsTable } from './kysely/tables/entityStatementsTable';
+import { type Insertable } from 'kysely';
+
+export default async function saveEntityStatements(
+  entity: Entity,
+  statementShas: string[],
+) {
+  if (statementShas.length === 0) return;
+
+  const dbEntityStatements: Insertable<EntityStatementsTable> =
+    statementShas.map((statementSha) => ({
+      project_id: entity.projectId,
+      entity_sha: entity.sha,
+      statement_sha: statementSha,
+    }));
+
+  try {
+    await getDatabase()
+      .insertInto('entity_statements')
+      .values(dbEntityStatements)
+      .executeTakeFirst();
+  } catch (error) {
+    console.log('Error saving entity statements', error);
+  }
+}

package/analyzer-template/packages/database/src/lib/scenarioToDb.ts

@@ -0,0 +1,35 @@
+import { Scenario } from '~codeyam/types';
+import { type Insertable } from 'kysely';
+import { ScenariosTable } from './kysely/tables/scenariosTable';
+import { randomUUID } from 'crypto';
+
+export default function scenarioToDb(
+  scenario: Scenario,
+): Insertable<ScenariosTable> {
+  const {
+    id,
+    projectId,
+    analysisId,
+    previousVersionId,
+    analysis,
+    metadata,
+    data,
+    ...remaining
+  } = scenario;
+
+  delete remaining.userScenarios;
+  delete remaining.comments;
+
+  if ('created_at' in remaining) {
+    delete remaining.created_at;
+  }
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    metadata: metadata ? JSON.stringify(metadata) : null,
+    project_id: projectId,
+    analysis_id: analysisId,
+    previous_version_id: previousVersionId,
+  };
+}

package/analyzer-template/packages/database/src/lib/updateCommitMetadata.ts

@@ -0,0 +1,187 @@
+/*
+ * updates commit metadata by merging partial updates with existing data.
+ * uses database transactions to ensure atomic updates and handle concurrent modifications.
+ */
+import {
+  Commit,
+  CommitMetadata,
+  CommitRunStatus,
+  DeepReadonly,
+} from '~codeyam/types';
+import { getDatabase } from './kysely/db';
+import { awsLog } from '~codeyam/utils';
+import { deepMerge, DeepUpdate } from './jsonUpdateUtils';
+import dbToCommit from './dbToCommit';
+
+type UpdateCommitMetadataArgs = {
+  commitId?: string;
+  commitSha?: string;
+  metadataUpdate?: DeepUpdate<CommitMetadata>;
+  runStatusUpdate?: DeepUpdate<CommitRunStatus>;
+  archiveCurrentRun?: boolean;
+  updateCallback?: (
+    metadata: CommitMetadata,
+    commit: DeepReadonly<Commit>,
+  ) => void | Promise<void>;
+};
+
+export default async function updateCommitMetadata({
+  commitId,
+  commitSha,
+  metadataUpdate,
+  runStatusUpdate,
+  archiveCurrentRun,
+  updateCallback,
+}: UpdateCommitMetadataArgs): Promise<CommitMetadata | null> {
+  try {
+    return await getDatabase()
+      .transaction()
+      .execute(async (trx) => {
+        const commit = await trx
+          .selectFrom('commits')
+          .selectAll()
+          .$if(!!commitId, (qb: any) => qb.where('id', '=', commitId))
+          .$if(!!commitSha, (qb: any) => qb.where('sha', '=', commitSha))
+          .executeTakeFirst();
+
+        if (!commit) {
+          awsLog(
+            `CodeYam Error: updateCommitMetadata(): Commit ${commitId} not found`,
+          );
+          return null;
+        }
+
+        // Initialize metadata if it doesn't exist (metadata is already parsed from DB)
+        const currentMetadata: CommitMetadata = commit.metadata || {};
+
+        // if the convenience parameter for latest run was supplied, merge it in
+        if (runStatusUpdate) {
+          runStatusUpdate.lastUpdatedAt ??= new Date().toISOString();
+
+          if (runStatusUpdate.currentEntityShas !== undefined) {
+            console.log(
+              `[updateCommitMetadata] Updating currentRun.currentEntityShas`,
+            );
+            console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
+            console.log(
+              `[updateCommitMetadata] Previous entity SHAs:`,
+              currentMetadata.currentRun?.currentEntityShas,
+            );
+            console.log(
+              `[updateCommitMetadata] New entity SHAs:`,
+              runStatusUpdate.currentEntityShas,
+            );
+            console.log(
+              `[updateCommitMetadata] Archive flag:`,
+              archiveCurrentRun,
+            );
+          }
+
+          metadataUpdate = deepMerge(metadataUpdate ?? {}, {
+            currentRun: runStatusUpdate,
+          });
+        } else if (!metadataUpdate && !updateCallback) {
+          return currentMetadata; // nothing to update
+        }
+
+        // Deep merge the baseline status updates
+        const updatedMetadata: CommitMetadata = metadataUpdate
+          ? deepMerge(currentMetadata, metadataUpdate)
+          : currentMetadata;
+
+        // Archive current run if requested
+        if (archiveCurrentRun && updatedMetadata.currentRun) {
+          console.log(
+            `[updateCommitMetadata] ========================================`,
+          );
+          console.log(`[updateCommitMetadata] ARCHIVING CURRENT RUN`);
+          console.log(`[updateCommitMetadata] Commit SHA: ${commitSha}`);
+          console.log(
+            `[updateCommitMetadata] Current run entity SHAs:`,
+            updatedMetadata.currentRun.currentEntityShas,
+          );
+          console.log(
+            `[updateCommitMetadata] Current run PIDs: analyzer=${updatedMetadata.currentRun.analyzerPid}, capture=${updatedMetadata.currentRun.capturePid}`,
+          );
+          console.log(
+            `[updateCommitMetadata] Current run completed: analyses=${updatedMetadata.currentRun.analysesCompleted}, captures=${updatedMetadata.currentRun.capturesCompleted}`,
+          );
+          console.log(
+            `[updateCommitMetadata] Historical runs before archiving: ${updatedMetadata.historicalRuns?.length || 0}`,
+          );
+
+          const runToArchive: CommitRunStatus = {
+            ...updatedMetadata.currentRun,
+            archivedAt: new Date().toISOString(),
+          };
+
+          console.log(
+            `[updateCommitMetadata] Run to archive:`,
+            JSON.stringify(runToArchive, null, 2),
+          );
+
+          updatedMetadata.historicalRuns = [
+            ...(updatedMetadata.historicalRuns || []),
+            runToArchive,
+          ];
+
+          console.log(
+            `[updateCommitMetadata] Historical runs after archiving: ${updatedMetadata.historicalRuns.length}`,
+          );
+          console.log(
+            `[updateCommitMetadata] All historical runs:`,
+            JSON.stringify(
+              updatedMetadata.historicalRuns.map((r) => ({
+                entityShas: r.currentEntityShas,
+                archivedAt: r.archivedAt,
+                completed: {
+                  analyses: r.analysesCompleted,
+                  captures: r.capturesCompleted,
+                },
+              })),
+              null,
+              2,
+            ),
+          );
+          console.log(
+            `[updateCommitMetadata] ========================================`,
+          );
+        }
+
+        // Allow callback to perform complex atomic updates
+        if (updateCallback) {
+          await updateCallback(updatedMetadata, dbToCommit(commit));
+        }
+
+        try {
+          const result = await trx
+            .updateTable('commits')
+            .set({ metadata: JSON.stringify(updatedMetadata) })
+            .where('id', '=', commit.id)
+            .returningAll()
+            .executeTakeFirst();
+
+          if (!result) {
+            awsLog(
+              `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
+            );
+            return currentMetadata;
+          }
+
+          return updatedMetadata;
+        } catch (error) {
+          awsLog(
+            `CodeYam Error: updateCommitMetadata(): Failed to update commit ${commitId}`,
+            error,
+          );
+          return currentMetadata;
+        }
+      });
+  } catch (error) {
+    awsLog(
+      `CodeYam Error: updateCommitMetadata(): Transaction failed for commit ${commitId}`,
+      error,
+    );
+    return null;
+  }
+}