@codeyam/codeyam-cli 0.1.0-staging.28f73cf → 0.1.0-staging.323686

package/analyzer-template/packages/database/src/lib/loadEntity.ts

@@ -0,0 +1,105 @@
+import { ExpressionBuilder } from 'kysely';
+import { getDatabase } from './kysely/db';
+import dbToEntity from './dbToEntity';
+import { awsLog } from '~codeyam/utils';
+import { getEntityBranchesAsJson } from './loadEntityBranches';
+
+export interface loadEntityArgs {
+  projectId: string;
+  sha: string;
+}
+
+export interface loadEntityByNameAndPathArgs {
+  projectId: string;
+  name: string;
+  filePath: string;
+}
+
+export default async function loadEntity({ projectId, sha }: loadEntityArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.sha', '=', sha)
+      .executeTakeFirst();
+
+    if (!entity) {
+      // Only log at debug level - Entity not found is expected in E2E baseline mode
+      // where entities are created fresh and don't exist in the database yet
+      if (process.env.CODEYAM_E2E_BASELINE_MODE !== 'true') {
+        awsLog('CodeYam Error: Load Entity: Entity not found', null, {
+          projectId,
+          sha,
+        });
+      }
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      sha,
+    });
+    return null;
+  }
+}
+
+/**
+ * Load entity by name and file path instead of SHA.
+ * This is useful for E2E tests where we want to avoid hardcoding SHAs.
+ */
+export async function loadEntityByNameAndPath({
+  projectId,
+  name,
+  filePath,
+}: loadEntityByNameAndPathArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.name', '=', name)
+      .where('files.path', '=', filePath)
+      .executeTakeFirst();
+
+    if (!entity) {
+      awsLog(
+        'CodeYam Error: Load Entity: Entity not found by name and path',
+        null,
+        {
+          projectId,
+          name,
+          filePath,
+        },
+      );
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      name,
+      filePath,
+    });
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadEntityBranches.ts

@@ -0,0 +1,178 @@
+import { EntityBranch } from '~codeyam/types';
+import { getDatabase, getJsonHelper } from './kysely/db';
+import dbToEntityBranch from './dbToEntityBranch';
+import { DbCommit, DbEntityBranch } from './kysely/tableRelations';
+import { getCommitAuthorsAsJson } from './loadCommit';
+import {
+  JsonSelectBuilder,
+  JsonSelectConstraint,
+} from './kysely/aggregationHelpers';
+import { EntityBranchesTableColumns } from './kysely/tables/entityBranchesTable';
+import { BackgroundJobsTableColumns } from './kysely/tables/backgroundJobsTable';
+import { ExpressionBuilder } from 'kysely';
+
+interface LoadEntityBranchArgs {
+  filePath?: string;
+  entityName?: string;
+  entitySha?: string;
+  active?: boolean;
+  branchId?: string;
+}
+
+export default async function loadEntityBranches({
+  filePath,
+  entityName,
+  entitySha,
+  active,
+  branchId,
+}: LoadEntityBranchArgs): Promise<EntityBranch[] | null> {
+  const db = getDatabase();
+
+  try {
+    // Build the main query with explicit joins
+    let query = db
+      .selectFrom('entity_branches')
+      .innerJoin('entities', 'entity_branches.entity_sha', 'entities.sha')
+      .selectAll('entity_branches')
+      .selectAll('entities');
+
+    // Apply filters
+    if (filePath) {
+      query = query.where('entities.file_path', '=', filePath);
+    }
+
+    if (entityName) {
+      query = query.where('entities.name', '=', entityName);
+    }
+
+    if (entitySha) {
+      query = query.where('entity_branches.entity_sha', '=', entitySha);
+    }
+
+    if (active !== undefined) {
+      query = query.where('entity_branches.active', '=', active);
+    }
+
+    if (branchId) {
+      query = query.where('entity_branches.branch_id', '=', branchId);
+    }
+
+    const results = await query.execute();
+
+    if (!results || results.length === 0) {
+      return null;
+    }
+
+    // Group results by entity to load commits efficiently
+    const entityIds = [
+      ...new Set(results.map((r) => r.commit_id).filter(Boolean)),
+    ];
+    const commitsMap = await loadCommitsForEntities(entityIds);
+
+    // Build DbEntityBranch objects with full entity and commit data
+    const dbEntityBranches: DbEntityBranch[] = results.map((result) => {
+      const entityBranch = {
+        entity_sha: result.entity_sha,
+        branch_id: result.branch_id,
+        active: !!result.active,
+      };
+
+      const entity = {
+        sha: result.sha,
+        project_id: result.project_id,
+        file_id: result.file_id,
+        name: result.name,
+        entity_type: result.entity_type,
+        file_path: result.file_path,
+        description: result.description,
+        documentation: result.documentation,
+        metadata: result.metadata,
+        quality: result.quality,
+        commit_id: result.commit_id,
+        created_at: result.created_at,
+        updated_at: result.updated_at,
+        commit: result.commit_id ? commitsMap.get(result.commit_id) : undefined,
+      };
+
+      return {
+        ...entityBranch,
+        entity,
+      };
+    });
+
+    return dbEntityBranches.map(dbToEntityBranch);
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading entity branch',
+      { filePath, entityName, entitySha, branchId },
+      error,
+    );
+    return null;
+  }
+}
+
+async function loadCommitsForEntities(
+  commitIds: string[],
+): Promise<Map<string, DbCommit>> {
+  if (commitIds.length === 0) {
+    return new Map();
+  }
+
+  const db = getDatabase();
+
+  const { jsonArrayFrom } = getJsonHelper();
+  const startTime = Date.now();
+
+  try {
+    const commits = await db
+      .selectFrom('commits')
+      .selectAll('commits')
+      .select((eb: ExpressionBuilder<any, any>) => [
+        getCommitAuthorsAsJson(eb).as('author'),
+        jsonArrayFrom(
+          eb
+            .selectFrom('background_jobs')
+            .select(BackgroundJobsTableColumns)
+            .whereRef('background_jobs.commit_id', '=', 'commits.id'),
+        ).as('background_jobs'),
+      ])
+      .where('commits.id', 'in', commitIds)
+      .execute();
+    const queryTime = Date.now() - startTime;
+
+    // DEBUG: Log slow queries (includes files column which can be 1MB+)
+    if (queryTime > 100) {
+      const totalFilesSize = commits.reduce((sum, c: any) => {
+        return sum + (c.files ? JSON.stringify(c.files).length : 0);
+      }, 0);
+      console.log(
+        `CodeYam DEBUG: [CommitFilesTiming] loadCommitsForEntities took ${queryTime}ms (${commits.length} commits, totalFiles: ${Math.round(totalFilesSize / 1024)}KB)`,
+      );
+    }
+
+    return new Map(commits.map((commit) => [commit.id, commit]));
+  } catch (error) {
+    console.log('CodeYam Error: Loading commits for entities', error, {
+      commitIds,
+    });
+    return new Map();
+  }
+}
+
+// Export for reuse in other loaders
+export function getEntityBranchesAsJson(
+  eb: JsonSelectBuilder,
+  constraint?: JsonSelectConstraint,
+) {
+  const { jsonArrayFrom } = getJsonHelper();
+
+  let query = eb
+    .selectFrom('entity_branches')
+    .select(EntityBranchesTableColumns);
+
+  if (constraint) {
+    query = constraint(query);
+  }
+
+  return jsonArrayFrom(query);
+}

package/analyzer-template/packages/database/src/lib/loadFile.ts

@@ -0,0 +1,42 @@
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+
+export default async function loadFile({
+  projectId,
+  id,
+  path,
+}: {
+  projectId?: string;
+  id?: string;
+  path?: string;
+}) {
+  const db = getDatabase();
+
+  let query = db.selectFrom('files').selectAll();
+
+  if (projectId) {
+    query = query.where('project_id', '=', projectId);
+  }
+
+  if (id) {
+    query = query.where('id', '=', id);
+  }
+
+  if (path) {
+    query = query.where('path', '=', path);
+  }
+
+  try {
+    const file = await query.executeTakeFirst();
+
+    if (!file) {
+      console.log('Error loading file', { projectId, id, path }, 'Not found');
+      return null;
+    }
+
+    return dbToFile(file);
+  } catch (error) {
+    console.log('Error loading file', { projectId, id, path }, error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFiles.ts

@@ -0,0 +1,134 @@
+import { File } from '~codeyam/types';
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+import { Selectable } from 'kysely';
+import { FilesTable } from './kysely/tables/filesTable';
+
+const RANGE_LIMIT = 1000;
+
+/**
+ * Load only file paths for a project (lightweight version)
+ * Use this when you only need paths, not full file objects (~80% memory savings)
+ */
+export async function loadFilePaths(projectId: string): Promise<string[]> {
+  const db = getDatabase();
+  const paths: string[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      const query = db
+        .selectFrom('files')
+        .select('path') // Only select path column
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      paths.push(...filesData.map((f) => f.path));
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return paths;
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading file paths in loadFilePaths',
+      error,
+    );
+    return [];
+  }
+}
+
+interface LoadFilesArgs {
+  projectId: string;
+  filePaths?: string[];
+  fileIds?: string[];
+  fileNames?: string[];
+}
+
+export default async function loadFiles({
+  projectId,
+  filePaths,
+  fileIds,
+  fileNames,
+}: LoadFilesArgs): Promise<File[] | null> {
+  if (filePaths && filePaths.length > 50) {
+    const allFiles: File[] = [];
+    for (let i = 0; i < filePaths.length; i += 50) {
+      const chunk = filePaths.slice(i, i + 50);
+      const chunkFiles = await loadFiles({
+        projectId,
+        filePaths: chunk,
+        fileIds,
+        fileNames,
+      });
+
+      if (chunkFiles) {
+        allFiles.push(...chunkFiles);
+      }
+    }
+    return allFiles;
+  }
+
+  const db = getDatabase();
+  const files: Selectable<FilesTable>[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      let query = db
+        .selectFrom('files')
+        .selectAll()
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      if (filePaths) {
+        if (filePaths.length === 0) return [];
+        query = query.where('path', 'in', filePaths);
+      }
+
+      if (fileIds) {
+        if (fileIds.length === 0) return [];
+        query = query.where('id', 'in', fileIds);
+      }
+
+      if (fileNames) {
+        if (fileNames.length === 0) return [];
+        query = query.where('name', 'in', fileNames);
+      }
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      files.push(...filesData);
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return files?.map(dbToFile);
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading project files in loadFiles',
+      error,
+    );
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadReadyToBeCapturedAnalyses.ts

@@ -0,0 +1,101 @@
+import { Analysis } from '~codeyam/types';
+import dbToAnalysis from './dbToAnalysis';
+import { getDatabase, getJsonHelper } from './kysely/db';
+import { sql, ExpressionBuilder } from 'kysely';
+import { aliasedTableColumns } from './kysely/aggregationHelpers';
+import { EntitiesTableColumns } from './kysely/tables/entitiesTable';
+import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+import { DbAnalysis } from './kysely/tableRelations';
+
+export interface LoadReadyToBeCapturedAnalysesArgs {
+  projectId?: string;
+  ids?: string[];
+  branchCommitSha?: string;
+}
+
+export default async function loadReadyToBeCapturedAnalyses(
+  params: LoadReadyToBeCapturedAnalysesArgs,
+): Promise<Analysis[]> {
+  const { projectId, branchCommitSha, ids } = params;
+
+  const db = getDatabase();
+  const { jsonObjectFrom, jsonArrayFrom } = getJsonHelper();
+
+  let query = db
+    .selectFrom('analyses')
+    .innerJoin('entities', 'entities.sha', 'analyses.entity_sha')
+    .selectAll('analyses')
+    .select((eb: ExpressionBuilder<any, any>) => [
+      jsonObjectFrom(
+        eb
+          .selectFrom('entities')
+          .select(aliasedTableColumns('entities', EntitiesTableColumns))
+          .whereRef('entities.sha', '=', 'analyses.entity_sha')
+          .limit(1),
+      ).as('entity'),
+      jsonArrayFrom(
+        eb
+          .selectFrom('scenarios')
+          .select(aliasedTableColumns('scenarios', ScenariosTableColumns))
+          .whereRef('scenarios.analysis_id', '=', 'analyses.id'),
+      ).as('scenarios'),
+    ])
+    // Cross-database compatible: use -> (not ->>) for boolean JSON fields
+    // PostgreSQL: jsonb `true` auto-casts to string 'true' for comparison
+    // SQLite: -> returns text 'true' which equals string 'true'
+    // (Note: ->> returns integer 1 in SQLite for booleans, breaking string comparisons)
+    .where(
+      sql<string>`${sql.ref('analyses.status')} -> 'readyToBeCaptured'`,
+      '=',
+      'true',
+    )
+    // Exclude entities that are not exported - they cannot be captured because they can't be imported
+    // Use -> operator for cross-database boolean comparison
+    .where(
+      sql<boolean>`(${sql.ref('entities.metadata')} -> 'notExported') IS NULL OR (${sql.ref('entities.metadata')} -> 'notExported') != 'true'`,
+    );
+
+  if (projectId) {
+    query = query.where('analyses.project_id', '=', projectId);
+  }
+
+  // Only filter by branchCommitSha if specific IDs are NOT provided
+  // When specific analysis IDs are requested (e.g., debug/recapture), we want those exact analyses
+  // regardless of their branch_commit_sha (which may be NULL in CLI scenarios)
+  if (branchCommitSha && (!ids || ids.length === 0)) {
+    query = query.where('analyses.branch_commit_sha', '=', branchCommitSha);
+  }
+
+  if (ids && ids.length > 0) {
+    query = query.where('analyses.id', 'in', ids);
+  }
+
+  const results = await query.execute();
+
+  if (!results || results.length === 0) {
+    console.log('CodeYam: No analyses found for readyToBeCaptured', {
+      projectId,
+      branchCommitSha,
+      ids: ids?.length,
+    });
+    return [];
+  }
+
+  const analyses = results as DbAnalysis[];
+  const mappedAnalyses = analyses.map(dbToAnalysis);
+
+  // Log entity export status for debugging notExported filtering
+  console.log(
+    'CodeYam: loadReadyToBeCapturedAnalyses results (notExported entities should be filtered)',
+    {
+      count: mappedAnalyses.length,
+      entities: mappedAnalyses.map((a) => ({
+        name: a.entityName,
+        filePath: a.filePath,
+        notExported: a.entity?.metadata?.notExported ?? false,
+      })),
+    },
+  );
+
+  return mappedAnalyses;
+}

package/analyzer-template/packages/database/src/lib/loadStatement.ts

@@ -0,0 +1,23 @@
+import { getDatabase } from './kysely/db';
+import { DbStatement } from '~codeyam/database';
+
+export default async function loadStatement(
+  sha: string,
+): Promise<Pick<DbStatement, 'sha' | 'results'> | null> {
+  try {
+    const result = await getDatabase()
+      .selectFrom('statements')
+      .select(['sha', 'results'])
+      .where('sha', '=', sha)
+      .executeTakeFirst();
+    // for unknown reasons, SQLite parses the JSON in 'results' even though
+    // the field is defined as text. We need to stringify it back.
+    if (result && result.results && typeof result.results !== 'string') {
+      result.results = JSON.stringify(result.results);
+    }
+    return result ?? null;
+  } catch (error) {
+    console.log('Error loading statement', error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/saveEntityStatements.ts

@@ -0,0 +1,27 @@
+import { Entity } from '~codeyam/types';
+import { getDatabase } from './kysely/db';
+import { EntityStatementsTable } from './kysely/tables/entityStatementsTable';
+import { type Insertable } from 'kysely';
+
+export default async function saveEntityStatements(
+  entity: Entity,
+  statementShas: string[],
+) {
+  if (statementShas.length === 0) return;
+
+  const dbEntityStatements: Insertable<EntityStatementsTable> =
+    statementShas.map((statementSha) => ({
+      project_id: entity.projectId,
+      entity_sha: entity.sha,
+      statement_sha: statementSha,
+    }));
+
+  try {
+    await getDatabase()
+      .insertInto('entity_statements')
+      .values(dbEntityStatements)
+      .executeTakeFirst();
+  } catch (error) {
+    console.log('Error saving entity statements', error);
+  }
+}

package/analyzer-template/packages/database/src/lib/scenarioToDb.ts

@@ -0,0 +1,35 @@
+import { Scenario } from '~codeyam/types';
+import { type Insertable } from 'kysely';
+import { ScenariosTable } from './kysely/tables/scenariosTable';
+import { randomUUID } from 'crypto';
+
+export default function scenarioToDb(
+  scenario: Scenario,
+): Insertable<ScenariosTable> {
+  const {
+    id,
+    projectId,
+    analysisId,
+    previousVersionId,
+    analysis,
+    metadata,
+    data,
+    ...remaining
+  } = scenario;
+
+  delete remaining.userScenarios;
+  delete remaining.comments;
+
+  if ('created_at' in remaining) {
+    delete remaining.created_at;
+  }
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    metadata: metadata ? JSON.stringify(metadata) : null,
+    project_id: projectId,
+    analysis_id: analysisId,
+    previous_version_id: previousVersionId,
+  };
+}