@codeyam/codeyam-cli 0.1.0-staging.e38f7bd → 0.1.0-staging.f777668

package/analyzer-template/packages/database/src/lib/loadAnalyses.ts

@@ -27,6 +27,7 @@ function buildAnalysesQuery({
   commitIds,
   branchCommitSha,
   limit,
+  excludeMetadata,
 }: LoadAnalysesArgs & {
   ids?: string[];
   fileIds?: string[];
@@ -37,7 +38,32 @@ function buildAnalysesQuery({
   const { jsonObjectFrom, jsonArrayFrom } = getJsonHelper();
 
   // Create CTE for filtering analyses
-  let filteredAnalysisCTE = db.selectFrom('analyses').selectAll('analyses');
+  // When excludeMetadata is true, select specific columns to avoid loading 64MB+ of metadata
+  let filteredAnalysisCTE = excludeMetadata
+    ? db
+        .selectFrom('analyses')
+        .select([
+          'analyses.id',
+          'analyses.project_id',
+          'analyses.file_id',
+          'analyses.commit_id',
+          'analyses.entity_sha',
+          'analyses.entity_name',
+          'analyses.entity_type',
+          'analyses.file_path',
+          'analyses.status',
+          'analyses.created_at',
+          'analyses.updated_at',
+          'analyses.tree_sha',
+          'analyses.analyzed_tree_sha',
+          'analyses.dependency_analyzed_tree_sha',
+          'analyses.previous_analysis_id',
+          'analyses.branch_commit_sha',
+          'analyses.indirect',
+          'analyses.committed_at',
+          'analyses.completed_at',
+        ])
+    : db.selectFrom('analyses').selectAll('analyses');
 
   // Apply filters
   if (projectId) {
@@ -96,6 +122,35 @@ function buildAnalysesQuery({
   }
 
   // Main query with JSON aggregations referencing the CTE
+  // When excludeMetadata is true, skip the entity subquery (saves loading entity metadata again)
+  if (excludeMetadata) {
+    return db
+      .with('filtered_analyses', () => filteredAnalysisCTE)
+      .selectFrom('filtered_analyses')
+      .selectAll('filtered_analyses')
+      .select((eb: ExpressionBuilder<any, any>) => [
+        // Skip entity subquery - not needed for list views and loads 41MB+ of entity metadata
+
+        jsonArrayFrom(
+          eb
+            .selectFrom('scenarios')
+            .select(aliasedTableColumns('scenarios', ScenariosTableColumns))
+            .whereRef('scenarios.analysis_id', '=', 'filtered_analyses.id'),
+        ).as('scenarios'),
+
+        jsonArrayFrom(
+          eb
+            .selectFrom('analysis_branches')
+            .select(['id', 'branch_id'])
+            .whereRef(
+              'analysis_branches.analysis_id',
+              '=',
+              'filtered_analyses.id',
+            ),
+        ).as('analysis_branches'),
+      ]);
+  }
+
   return db
     .with('filtered_analyses', () => filteredAnalysisCTE)
     .selectFrom('filtered_analyses')
@@ -138,6 +193,8 @@ export interface LoadAnalysesArgs {
   entityShas?: string[];
   branchCommitSha?: string;
   limit?: number;
+  /** Exclude large metadata columns for faster list queries */
+  excludeMetadata?: boolean;
 }
 
 export default async function loadAnalyses(

package/analyzer-template/packages/database/src/lib/loadAnalysis.ts

@@ -45,6 +45,7 @@ export default async function loadAnalysis({
   // TODO: Consider refactoring to use CTE + aliasedTableColumns pattern (Pattern 3)
   // like loadMostRecentPreviousAnalysis for more elegant single-query approach
   const db = getDatabase();
+  const startTime = Date.now();
 
   try {
     // Build the main query
@@ -169,6 +170,8 @@ export default async function loadAnalysis({
     });
 
     const analysis = await query.executeTakeFirst();
+    const queryTime = Date.now() - startTime;
+
     if (!analysis) {
       awsLog('CodeYam Error: Analysis not found', null, {
         id,
@@ -188,6 +191,16 @@ export default async function loadAnalysis({
       return null;
     }
 
+    // DEBUG: Log slow queries (when includeCommitAndBranch=true, includes files column which can be 1MB+)
+    if (queryTime > 100 && includeCommitAndBranch) {
+      const commit = (analysis as any).commit;
+      const filesSize = commit?.files ? JSON.stringify(commit.files).length : 0;
+      console.log(
+        `CodeYam DEBUG: [CommitFilesTiming] loadAnalysis took ${queryTime}ms (files: ${Math.round(filesSize / 1024)}KB)`,
+        { id: analysis.id, entityName: analysis.entity_name },
+      );
+    }
+
     return dbToAnalysis(analysis);
   } catch (error) {
     awsLog('CodeYam Error: Database error loading analysis', error, {

package/analyzer-template/packages/database/src/lib/loadBranch.ts

@@ -98,9 +98,10 @@ async function loadCommitsForBranch(branchId: string): Promise<DbCommit[]> {
   const db = getDatabase();
 
   const { jsonArrayFrom } = getJsonHelper();
+  const startTime = Date.now();
 
   try {
-    return await db
+    const commits = await db
       .selectFrom('commits')
       .selectAll()
       .select((eb: ExpressionBuilder<any, any>) => [
@@ -115,6 +116,20 @@ async function loadCommitsForBranch(branchId: string): Promise<DbCommit[]> {
       .where('commits.branch_id', '=', branchId)
       .orderBy('commits.committed_at', 'desc')
       .execute();
+    const queryTime = Date.now() - startTime;
+
+    // DEBUG: Log slow queries (includes files column which can be 1MB+)
+    if (queryTime > 100) {
+      const totalFilesSize = commits.reduce((sum, c: any) => {
+        return sum + (c.files ? JSON.stringify(c.files).length : 0);
+      }, 0);
+      console.log(
+        `CodeYam DEBUG: [CommitFilesTiming] loadCommitsForBranch took ${queryTime}ms (${commits.length} commits, totalFiles: ${Math.round(totalFilesSize / 1024)}KB)`,
+        { branchId },
+      );
+    }
+
+    return commits;
   } catch (error) {
     awsLog('CodeYam Error: Loading commits for branch', error, { branchId });
     return [];

package/analyzer-template/packages/database/src/lib/loadCommit.ts

@@ -69,6 +69,7 @@ export default async function loadCommit({
   branchId,
 }: LoadCommitArgs) {
   const db = getDatabase();
+  const startTime = Date.now();
 
   try {
     let query = db
@@ -97,6 +98,7 @@ export default async function loadCommit({
     query = query.orderBy('committed_at', 'desc').limit(1);
 
     const commit = await query.executeTakeFirst();
+    const queryTime = Date.now() - startTime;
 
     if (!commit) {
       awsLog('CodeYam Error: Commit not found', null, {
@@ -108,6 +110,15 @@ export default async function loadCommit({
       return null;
     }
 
+    // DEBUG: Log slow queries (includes files column which can be 1MB+)
+    if (queryTime > 100) {
+      const filesSize = commit.files ? JSON.stringify(commit.files).length : 0;
+      console.log(
+        `CodeYam DEBUG: [CommitFilesTiming] loadCommit took ${queryTime}ms (files: ${Math.round(filesSize / 1024)}KB)`,
+        { sha: commit.sha },
+      );
+    }
+
     // Load branch and mergedBranch separately
     const [branch, mergedBranch, analyses] = await Promise.all([
       commit.branch_id

package/analyzer-template/packages/database/src/lib/loadCommits.ts

@@ -9,6 +9,7 @@ import {
 import { awsLog } from '~codeyam/utils';
 import { Commit } from '~codeyam/types';
 import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+import { CommitsTableColumnsLite } from './kysely/tables/commitsTable';
 import { sql, ExpressionBuilder } from 'kysely';
 
 interface LoadCommitsArgs {
@@ -19,6 +20,8 @@ interface LoadCommitsArgs {
   shas?: string[];
   fileNames?: string[];
   limit?: number;
+  /** Skip loading related data (analyses, entities, branches) for faster queries */
+  skipRelations?: boolean;
 }
 
 async function loadBranchesForCommits(
@@ -137,6 +140,7 @@ export default async function loadCommits({
   shas,
   fileNames,
   limit = 10,
+  skipRelations = false,
 }: LoadCommitsArgs): Promise<Commit[]> {
   if (!projectId && !ids) {
     throw new Error('Must provide projectId or ids');
@@ -144,27 +148,38 @@ export default async function loadCommits({
 
   const db = getDatabase();
   const { jsonObjectFrom } = getJsonHelper();
+  const startTime = Date.now();
 
   try {
-    let query = db
-      .selectFrom('commits')
-      .selectAll('commits')
-      .select((eb: ExpressionBuilder<any, any>) => [
-        jsonObjectFrom(
-          eb
-            .selectFrom('github_users')
-            .select([
-              'username',
-              'preferred_username as preferredUsername',
-              'avatar_url as avatarUrl',
-            ])
-            .where(
-              'github_users.username',
-              '=',
-              eb.ref('commits.author_github_username'),
-            ),
-        ).as('author'),
-      ]);
+    // When skipRelations is true, use lite columns (excludes 'files' which can be 1MB+ per commit)
+    // and skip the author subquery for maximum efficiency
+    let query: any;
+    if (skipRelations) {
+      const liteColumns = CommitsTableColumnsLite.map(
+        (col) => `commits.${col}` as any,
+      );
+      query = db.selectFrom('commits').select(liteColumns);
+    } else {
+      query = db
+        .selectFrom('commits')
+        .selectAll('commits')
+        .select((eb: ExpressionBuilder<any, any>) => [
+          jsonObjectFrom(
+            eb
+              .selectFrom('github_users')
+              .select([
+                'username',
+                'preferred_username as preferredUsername',
+                'avatar_url as avatarUrl',
+              ])
+              .where(
+                'github_users.username',
+                '=',
+                eb.ref('commits.author_github_username'),
+              ),
+          ).as('author'),
+        ]);
+    }
 
     if (projectId) {
       query = query.where('project_id', '=', projectId);
@@ -203,11 +218,35 @@ export default async function loadCommits({
       .orderBy('committed_at', 'desc')
       .limit(limit)
       .execute();
+    const queryTime = Date.now() - startTime;
 
     if (!commits || commits.length === 0) {
       return [];
     }
 
+    // DEBUG: Log slow queries (includes files column which can be 1MB+)
+    if (queryTime > 100) {
+      const totalFilesSize = commits.reduce((sum: number, c: any) => {
+        return sum + (c.files ? JSON.stringify(c.files).length : 0);
+      }, 0);
+      console.log(
+        `CodeYam DEBUG: [CommitFilesTiming] loadCommits took ${queryTime}ms (${commits.length} commits, totalFiles: ${Math.round(totalFilesSize / 1024)}KB)`,
+      );
+    }
+
+    // When skipRelations is true, just return commits without loading related data
+    // This is much faster for queries that only need commit metadata
+    if (skipRelations) {
+      const dbCommits: DbCommit[] = commits.map((commit: any) => ({
+        ...commit,
+        branch: undefined,
+        mergedBranch: undefined,
+        analyses: [],
+        entities: [],
+      }));
+      return dbCommits.map(dbToCommit);
+    }
+
     const commitIds = commits.map((c: any) => c.id);
 
     // Load all relations in parallel

package/analyzer-template/packages/database/src/lib/loadEntities.ts

@@ -9,6 +9,8 @@ export interface LoadEntitiesArgs {
   filePaths?: string[];
   names?: string[];
   shas?: string[];
+  /** Exclude large metadata column for faster list queries */
+  excludeMetadata?: boolean;
 }
 
 export default async function loadEntities({
@@ -18,6 +20,7 @@ export default async function loadEntities({
   filePaths,
   names,
   shas,
+  excludeMetadata,
 }: LoadEntitiesArgs): Promise<Entity[] | null> {
   if (
     (fileIds && fileIds.length == 0) ||
@@ -39,6 +42,7 @@ export default async function loadEntities({
         filePaths,
         names,
         shas: chunk,
+        excludeMetadata,
       });
 
       if (chunkEntities) {
@@ -51,9 +55,28 @@ export default async function loadEntities({
   const db = getDatabase();
 
   try {
-    const query = db
-      .selectFrom('entities')
-      .selectAll('entities')
+    // When excludeMetadata is true, select all columns except the large metadata column
+    // This dramatically speeds up list queries (41MB+ of JSON avoided)
+    const baseQuery = excludeMetadata
+      ? db
+          .selectFrom('entities')
+          .select([
+            'entities.project_id',
+            'entities.file_id',
+            'entities.commit_id',
+            'entities.name',
+            'entities.sha',
+            'entities.entity_type',
+            'entities.file_path',
+            'entities.description',
+            'entities.documentation',
+            'entities.quality',
+            'entities.created_at',
+            'entities.updated_at',
+          ])
+      : db.selectFrom('entities').selectAll('entities');
+
+    const query = baseQuery
       .$if(!!branchId, (qb) =>
         qb
           .innerJoin(

package/analyzer-template/packages/database/src/lib/loadEntityBranches.ts

@@ -121,6 +121,7 @@ async function loadCommitsForEntities(
   const db = getDatabase();
 
   const { jsonArrayFrom } = getJsonHelper();
+  const startTime = Date.now();
 
   try {
     const commits = await db
@@ -137,6 +138,17 @@ async function loadCommitsForEntities(
       ])
       .where('commits.id', 'in', commitIds)
       .execute();
+    const queryTime = Date.now() - startTime;
+
+    // DEBUG: Log slow queries (includes files column which can be 1MB+)
+    if (queryTime > 100) {
+      const totalFilesSize = commits.reduce((sum, c: any) => {
+        return sum + (c.files ? JSON.stringify(c.files).length : 0);
+      }, 0);
+      console.log(
+        `CodeYam DEBUG: [CommitFilesTiming] loadCommitsForEntities took ${queryTime}ms (${commits.length} commits, totalFiles: ${Math.round(totalFilesSize / 1024)}KB)`,
+      );
+    }
 
     return new Map(commits.map((commit) => [commit.id, commit]));
   } catch (error) {

package/analyzer-template/packages/database/src/lib/loadReadyToBeCapturedAnalyses.ts

@@ -23,6 +23,7 @@ export default async function loadReadyToBeCapturedAnalyses(
 
   let query = db
     .selectFrom('analyses')
+    .innerJoin('entities', 'entities.sha', 'analyses.entity_sha')
     .selectAll('analyses')
     .select((eb: ExpressionBuilder<any, any>) => [
       jsonObjectFrom(
@@ -39,25 +40,29 @@ export default async function loadReadyToBeCapturedAnalyses(
           .whereRef('scenarios.analysis_id', '=', 'analyses.id'),
       ).as('scenarios'),
     ])
+    // Cross-database compatible: use -> (not ->>) for boolean JSON fields
+    // PostgreSQL: jsonb `true` auto-casts to string 'true' for comparison
+    // SQLite: -> returns text 'true' which equals string 'true'
+    // (Note: ->> returns integer 1 in SQLite for booleans, breaking string comparisons)
     .where(
-      sql<string>` ${sql.ref('status')} -> 'readyToBeCaptured'`,
+      sql<string>`${sql.ref('analyses.status')} -> 'readyToBeCaptured'`,
       '=',
       'true',
     );
 
   if (projectId) {
-    query = query.where('project_id', '=', projectId);
+    query = query.where('analyses.project_id', '=', projectId);
   }
 
   // Only filter by branchCommitSha if specific IDs are NOT provided
   // When specific analysis IDs are requested (e.g., debug/recapture), we want those exact analyses
   // regardless of their branch_commit_sha (which may be NULL in CLI scenarios)
   if (branchCommitSha && (!ids || ids.length === 0)) {
-    query = query.where('branch_commit_sha', '=', branchCommitSha);
+    query = query.where('analyses.branch_commit_sha', '=', branchCommitSha);
   }
 
   if (ids && ids.length > 0) {
-    query = query.where('id', 'in', ids);
+    query = query.where('analyses.id', 'in', ids);
   }
 
   const results = await query.execute();
@@ -72,5 +77,20 @@ export default async function loadReadyToBeCapturedAnalyses(
   }
 
   const analyses = results as DbAnalysis[];
-  return analyses.map(dbToAnalysis);
+  const mappedAnalyses = analyses.map(dbToAnalysis);
+
+  // Log entity export status for debugging notExported filtering
+  console.log(
+    'CodeYam: loadReadyToBeCapturedAnalyses results (notExported entities should be filtered)',
+    {
+      count: mappedAnalyses.length,
+      entities: mappedAnalyses.map((a) => ({
+        name: a.entityName,
+        filePath: a.filePath,
+        notExported: a.entity?.metadata?.notExported ?? false,
+      })),
+    },
+  );
+
+  return mappedAnalyses;
 }

package/analyzer-template/packages/database/src/lib/projectToDb.ts

@@ -1,7 +1,7 @@
 import { Project } from '~codeyam/types';
 import { type Insertable } from 'kysely';
 import { ProjectsTable } from './kysely/tables/projectsTable';
-import { randomUUID } from 'crypto';
+const randomUUID = () => crypto.randomUUID();
 
 export default function projectToDb(
   project: Project,

package/analyzer-template/packages/database/src/lib/saveFiles.ts

@@ -3,7 +3,7 @@ import dbToFile from './dbToFile';
 import { File } from '~codeyam/types';
 import { Insertable, Selectable } from 'kysely';
 import { FilesTable } from './kysely/tables/filesTable';
-import { randomUUID } from 'crypto';
+const randomUUID = () => crypto.randomUUID();
 
 const BATCH_SIZE = 100;
 

package/analyzer-template/packages/database/src/lib/scenarioToDb.ts

@@ -1,7 +1,7 @@
 import { Scenario } from '~codeyam/types';
 import { type Insertable } from 'kysely';
 import { ScenariosTable } from './kysely/tables/scenariosTable';
-import { randomUUID } from 'crypto';
+const randomUUID = () => crypto.randomUUID();
 
 export default function scenarioToDb(
   scenario: Scenario,