@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.1669d45

package/analyzer-template/packages/database/src/lib/loadCommit.ts

@@ -0,0 +1,162 @@
+import { Database, getDatabase, getJsonHelper } from './kysely/db';
+import dbToCommit from './dbToCommit';
+import {
+  DbAnalysis,
+  DbBranch,
+  DbCommit,
+  DbScenario,
+} from './kysely/tableRelations';
+import { awsLog } from '~codeyam/utils';
+import { ExpressionBuilder } from 'kysely';
+import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+
+interface LoadCommitArgs {
+  id?: string;
+  projectId?: string;
+  sha?: string;
+  branchId?: string;
+}
+
+async function loadBranchForCommit(branchId: string): Promise<DbBranch | null> {
+  const db = getDatabase();
+
+  try {
+    return (
+      (await db
+        .selectFrom('branches')
+        .selectAll()
+        .where('id', '=', branchId)
+        .executeTakeFirst()) || null
+    );
+  } catch (error) {
+    awsLog('CodeYam Error: Loading branch for commit', error, { branchId });
+    return null;
+  }
+}
+
+async function loadAnalysesForCommit(commitId: string): Promise<DbAnalysis[]> {
+  const db = getDatabase();
+
+  const { jsonArrayFrom } = getJsonHelper();
+
+  try {
+    const analyses = await db
+      .selectFrom('analyses')
+      .selectAll('analyses')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        jsonArrayFrom(
+          eb
+            .selectFrom('scenarios')
+            .select(ScenariosTableColumns)
+            .whereRef('scenarios.analysis_id', '=', 'analyses.id')
+            .$castTo<DbScenario>(),
+        ).as('scenarios'),
+      )
+      .where('commit_id', '=', commitId)
+      .execute();
+
+    return analyses;
+  } catch (error) {
+    awsLog('CodeYam Error: Loading analyses for commit', error, { commitId });
+    return [];
+  }
+}
+
+export default async function loadCommit({
+  id,
+  projectId,
+  sha,
+  branchId,
+}: LoadCommitArgs) {
+  const db = getDatabase();
+
+  try {
+    let query = db
+      .selectFrom('commits')
+      .selectAll('commits')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getCommitAuthorsAsJson(eb).as('author'),
+      );
+
+    if (id) {
+      query = query.where('id', '=', id);
+    }
+
+    if (sha) {
+      query = query.where('sha', '=', sha);
+    }
+
+    if (projectId) {
+      query = query.where('project_id', '=', projectId);
+    }
+
+    if (branchId) {
+      query = query.where('branch_id', '=', branchId);
+    }
+
+    query = query.orderBy('committed_at', 'desc').limit(1);
+
+    const commit = await query.executeTakeFirst();
+
+    if (!commit) {
+      awsLog('CodeYam Error: Commit not found', null, {
+        id,
+        projectId,
+        sha,
+        branchId,
+      });
+      return null;
+    }
+
+    // Load branch and mergedBranch separately
+    const [branch, mergedBranch, analyses] = await Promise.all([
+      commit.branch_id
+        ? loadBranchForCommit(commit.branch_id)
+        : Promise.resolve(null),
+      commit.merged_branch_id
+        ? loadBranchForCommit(commit.merged_branch_id)
+        : Promise.resolve(null),
+      loadAnalysesForCommit(commit.id),
+    ]);
+
+    // Construct the full DbCommit object
+    const dbCommit: DbCommit = {
+      ...commit,
+      branch: branch || undefined,
+      mergedBranch: mergedBranch || undefined,
+      analyses,
+    };
+
+    return dbToCommit(dbCommit);
+  } catch (error) {
+    awsLog('CodeYam Error: Database error loading commit', error, {
+      id,
+      projectId,
+      sha,
+      branchId,
+    });
+    return null;
+  }
+}
+
+// Export for reuse in other loaders
+export function getCommitAuthorsAsJson(
+  eb: ExpressionBuilder<Database, 'commits'>,
+) {
+  const { jsonObjectFrom } = getJsonHelper();
+
+  return jsonObjectFrom(
+    eb
+      .selectFrom('github_users')
+      .select([
+        'username',
+        'preferred_username as preferredUsername',
+        'avatar_url as avatarUrl',
+      ])
+      .where(
+        'github_users.username',
+        '=',
+        eb.ref('commits.author_github_username'),
+      ),
+  );
+}

package/analyzer-template/packages/database/src/lib/loadEntities.ts

@@ -0,0 +1,94 @@
+import { Entity } from '~codeyam/types';
+import dbToEntity from './dbToEntity';
+import { getDatabase } from './kysely/db';
+
+export interface LoadEntitiesArgs {
+  projectId?: string;
+  branchId?: string;
+  fileIds?: string[];
+  filePaths?: string[];
+  names?: string[];
+  shas?: string[];
+}
+
+export default async function loadEntities({
+  projectId,
+  branchId,
+  fileIds,
+  filePaths,
+  names,
+  shas,
+}: LoadEntitiesArgs): Promise<Entity[] | null> {
+  if (
+    (fileIds && fileIds.length == 0) ||
+    (filePaths && filePaths.length == 0) ||
+    (names && names.length == 0) ||
+    (shas && shas.length == 0)
+  ) {
+    return [];
+  }
+
+  if (shas && shas.length > 50) {
+    const allEntities: Entity[] = [];
+    for (let i = 0; i < shas.length; i += 50) {
+      const chunk = shas.slice(i, i + 50);
+      const chunkEntities = await loadEntities({
+        projectId,
+        branchId,
+        fileIds,
+        filePaths,
+        names,
+        shas: chunk,
+      });
+
+      if (chunkEntities) {
+        allEntities.push(...chunkEntities);
+      }
+    }
+    return allEntities;
+  }
+
+  const db = getDatabase();
+
+  try {
+    const query = db
+      .selectFrom('entities')
+      .selectAll('entities')
+      .$if(!!branchId, (qb) =>
+        qb
+          .innerJoin(
+            'entity_branches',
+            'entity_branches.entity_sha',
+            'entities.sha',
+          )
+          .where('entity_branches.branch_id', '=', branchId),
+      )
+      .$if(!!projectId, (qb) => qb.where('entities.project_id', '=', projectId))
+      .$if(!!shas, (qb) => qb.where('entities.sha', 'in', shas))
+      .$if(!!filePaths, (qb) => qb.where('entities.file_path', 'in', filePaths))
+      .$if(!!names, (qb) => qb.where('entities.name', 'in', names))
+      .$if(!!fileIds, (qb) => qb.where('entities.file_id', 'in', fileIds));
+
+    const entities = await query.execute();
+
+    if (!entities || entities.length === 0) {
+      console.log('Load Entities: No entities found', {
+        projectId,
+        fileIds,
+        filePaths,
+        shas,
+      });
+      return null;
+    }
+
+    return entities.map(dbToEntity);
+  } catch (error) {
+    console.log('Load Entities: Error occurred', error, {
+      projectId,
+      fileIds,
+      filePaths,
+      shas,
+    });
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadEntity.ts

@@ -0,0 +1,105 @@
+import { ExpressionBuilder } from 'kysely';
+import { getDatabase } from './kysely/db';
+import dbToEntity from './dbToEntity';
+import { awsLog } from '~codeyam/utils';
+import { getEntityBranchesAsJson } from './loadEntityBranches';
+
+export interface loadEntityArgs {
+  projectId: string;
+  sha: string;
+}
+
+export interface loadEntityByNameAndPathArgs {
+  projectId: string;
+  name: string;
+  filePath: string;
+}
+
+export default async function loadEntity({ projectId, sha }: loadEntityArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.sha', '=', sha)
+      .executeTakeFirst();
+
+    if (!entity) {
+      // Only log at debug level - Entity not found is expected in E2E baseline mode
+      // where entities are created fresh and don't exist in the database yet
+      if (process.env.CODEYAM_E2E_BASELINE_MODE !== 'true') {
+        awsLog('CodeYam Error: Load Entity: Entity not found', null, {
+          projectId,
+          sha,
+        });
+      }
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      sha,
+    });
+    return null;
+  }
+}
+
+/**
+ * Load entity by name and file path instead of SHA.
+ * This is useful for E2E tests where we want to avoid hardcoding SHAs.
+ */
+export async function loadEntityByNameAndPath({
+  projectId,
+  name,
+  filePath,
+}: loadEntityByNameAndPathArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.name', '=', name)
+      .where('files.path', '=', filePath)
+      .executeTakeFirst();
+
+    if (!entity) {
+      awsLog(
+        'CodeYam Error: Load Entity: Entity not found by name and path',
+        null,
+        {
+          projectId,
+          name,
+          filePath,
+        },
+      );
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      name,
+      filePath,
+    });
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFile.ts

@@ -0,0 +1,42 @@
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+
+export default async function loadFile({
+  projectId,
+  id,
+  path,
+}: {
+  projectId?: string;
+  id?: string;
+  path?: string;
+}) {
+  const db = getDatabase();
+
+  let query = db.selectFrom('files').selectAll();
+
+  if (projectId) {
+    query = query.where('project_id', '=', projectId);
+  }
+
+  if (id) {
+    query = query.where('id', '=', id);
+  }
+
+  if (path) {
+    query = query.where('path', '=', path);
+  }
+
+  try {
+    const file = await query.executeTakeFirst();
+
+    if (!file) {
+      console.log('Error loading file', { projectId, id, path }, 'Not found');
+      return null;
+    }
+
+    return dbToFile(file);
+  } catch (error) {
+    console.log('Error loading file', { projectId, id, path }, error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFiles.ts

@@ -0,0 +1,134 @@
+import { File } from '~codeyam/types';
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+import { Selectable } from 'kysely';
+import { FilesTable } from './kysely/tables/filesTable';
+
+const RANGE_LIMIT = 1000;
+
+/**
+ * Load only file paths for a project (lightweight version)
+ * Use this when you only need paths, not full file objects (~80% memory savings)
+ */
+export async function loadFilePaths(projectId: string): Promise<string[]> {
+  const db = getDatabase();
+  const paths: string[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      const query = db
+        .selectFrom('files')
+        .select('path') // Only select path column
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      paths.push(...filesData.map((f) => f.path));
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return paths;
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading file paths in loadFilePaths',
+      error,
+    );
+    return [];
+  }
+}
+
+interface LoadFilesArgs {
+  projectId: string;
+  filePaths?: string[];
+  fileIds?: string[];
+  fileNames?: string[];
+}
+
+export default async function loadFiles({
+  projectId,
+  filePaths,
+  fileIds,
+  fileNames,
+}: LoadFilesArgs): Promise<File[] | null> {
+  if (filePaths && filePaths.length > 50) {
+    const allFiles: File[] = [];
+    for (let i = 0; i < filePaths.length; i += 50) {
+      const chunk = filePaths.slice(i, i + 50);
+      const chunkFiles = await loadFiles({
+        projectId,
+        filePaths: chunk,
+        fileIds,
+        fileNames,
+      });
+
+      if (chunkFiles) {
+        allFiles.push(...chunkFiles);
+      }
+    }
+    return allFiles;
+  }
+
+  const db = getDatabase();
+  const files: Selectable<FilesTable>[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      let query = db
+        .selectFrom('files')
+        .selectAll()
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      if (filePaths) {
+        if (filePaths.length === 0) return [];
+        query = query.where('path', 'in', filePaths);
+      }
+
+      if (fileIds) {
+        if (fileIds.length === 0) return [];
+        query = query.where('id', 'in', fileIds);
+      }
+
+      if (fileNames) {
+        if (fileNames.length === 0) return [];
+        query = query.where('name', 'in', fileNames);
+      }
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      files.push(...filesData);
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return files?.map(dbToFile);
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading project files in loadFiles',
+      error,
+    );
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadReadyToBeCapturedAnalyses.ts

@@ -0,0 +1,101 @@
+import { Analysis } from '~codeyam/types';
+import dbToAnalysis from './dbToAnalysis';
+import { getDatabase, getJsonHelper } from './kysely/db';
+import { sql, ExpressionBuilder } from 'kysely';
+import { aliasedTableColumns } from './kysely/aggregationHelpers';
+import { EntitiesTableColumns } from './kysely/tables/entitiesTable';
+import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+import { DbAnalysis } from './kysely/tableRelations';
+
+export interface LoadReadyToBeCapturedAnalysesArgs {
+  projectId?: string;
+  ids?: string[];
+  branchCommitSha?: string;
+}
+
+export default async function loadReadyToBeCapturedAnalyses(
+  params: LoadReadyToBeCapturedAnalysesArgs,
+): Promise<Analysis[]> {
+  const { projectId, branchCommitSha, ids } = params;
+
+  const db = getDatabase();
+  const { jsonObjectFrom, jsonArrayFrom } = getJsonHelper();
+
+  let query = db
+    .selectFrom('analyses')
+    .innerJoin('entities', 'entities.sha', 'analyses.entity_sha')
+    .selectAll('analyses')
+    .select((eb: ExpressionBuilder<any, any>) => [
+      jsonObjectFrom(
+        eb
+          .selectFrom('entities')
+          .select(aliasedTableColumns('entities', EntitiesTableColumns))
+          .whereRef('entities.sha', '=', 'analyses.entity_sha')
+          .limit(1),
+      ).as('entity'),
+      jsonArrayFrom(
+        eb
+          .selectFrom('scenarios')
+          .select(aliasedTableColumns('scenarios', ScenariosTableColumns))
+          .whereRef('scenarios.analysis_id', '=', 'analyses.id'),
+      ).as('scenarios'),
+    ])
+    // Cross-database compatible: use -> (not ->>) for boolean JSON fields
+    // PostgreSQL: jsonb `true` auto-casts to string 'true' for comparison
+    // SQLite: -> returns text 'true' which equals string 'true'
+    // (Note: ->> returns integer 1 in SQLite for booleans, breaking string comparisons)
+    .where(
+      sql<string>`${sql.ref('analyses.status')} -> 'readyToBeCaptured'`,
+      '=',
+      'true',
+    )
+    // Exclude entities that are not exported - they cannot be captured because they can't be imported
+    // Use -> operator for cross-database boolean comparison
+    .where(
+      sql<boolean>`(${sql.ref('entities.metadata')} -> 'notExported') IS NULL OR (${sql.ref('entities.metadata')} -> 'notExported') != 'true'`,
+    );
+
+  if (projectId) {
+    query = query.where('analyses.project_id', '=', projectId);
+  }
+
+  // Only filter by branchCommitSha if specific IDs are NOT provided
+  // When specific analysis IDs are requested (e.g., debug/recapture), we want those exact analyses
+  // regardless of their branch_commit_sha (which may be NULL in CLI scenarios)
+  if (branchCommitSha && (!ids || ids.length === 0)) {
+    query = query.where('analyses.branch_commit_sha', '=', branchCommitSha);
+  }
+
+  if (ids && ids.length > 0) {
+    query = query.where('analyses.id', 'in', ids);
+  }
+
+  const results = await query.execute();
+
+  if (!results || results.length === 0) {
+    console.log('CodeYam: No analyses found for readyToBeCaptured', {
+      projectId,
+      branchCommitSha,
+      ids: ids?.length,
+    });
+    return [];
+  }
+
+  const analyses = results as DbAnalysis[];
+  const mappedAnalyses = analyses.map(dbToAnalysis);
+
+  // Log entity export status for debugging notExported filtering
+  console.log(
+    'CodeYam: loadReadyToBeCapturedAnalyses results (notExported entities should be filtered)',
+    {
+      count: mappedAnalyses.length,
+      entities: mappedAnalyses.map((a) => ({
+        name: a.entityName,
+        filePath: a.filePath,
+        notExported: a.entity?.metadata?.notExported ?? false,
+      })),
+    },
+  );
+
+  return mappedAnalyses;
+}

package/analyzer-template/packages/database/src/lib/loadStatement.ts

@@ -0,0 +1,23 @@
+import { getDatabase } from './kysely/db';
+import { DbStatement } from '~codeyam/database';
+
+export default async function loadStatement(
+  sha: string,
+): Promise<Pick<DbStatement, 'sha' | 'results'> | null> {
+  try {
+    const result = await getDatabase()
+      .selectFrom('statements')
+      .select(['sha', 'results'])
+      .where('sha', '=', sha)
+      .executeTakeFirst();
+    // for unknown reasons, SQLite parses the JSON in 'results' even though
+    // the field is defined as text. We need to stringify it back.
+    if (result && result.results && typeof result.results !== 'string') {
+      result.results = JSON.stringify(result.results);
+    }
+    return result ?? null;
+  } catch (error) {
+    console.log('Error loading statement', error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/saveEntityStatements.ts

@@ -0,0 +1,27 @@
+import { Entity } from '~codeyam/types';
+import { getDatabase } from './kysely/db';
+import { EntityStatementsTable } from './kysely/tables/entityStatementsTable';
+import { type Insertable } from 'kysely';
+
+export default async function saveEntityStatements(
+  entity: Entity,
+  statementShas: string[],
+) {
+  if (statementShas.length === 0) return;
+
+  const dbEntityStatements: Insertable<EntityStatementsTable> =
+    statementShas.map((statementSha) => ({
+      project_id: entity.projectId,
+      entity_sha: entity.sha,
+      statement_sha: statementSha,
+    }));
+
+  try {
+    await getDatabase()
+      .insertInto('entity_statements')
+      .values(dbEntityStatements)
+      .executeTakeFirst();
+  } catch (error) {
+    console.log('Error saving entity statements', error);
+  }
+}