@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.596f0eb

package/analyzer-template/packages/database/src/lib/kysely/tableRelations.ts

@@ -0,0 +1,108 @@
+import { type Selectable } from 'kysely';
+import { UserScenariosTable } from './tables/userScenariosTable';
+import { AnalysesTable } from './tables/analysesTable';
+import { AnalysisBranchesTable } from './tables/analysisBranchesTable';
+import { BackgroundJobsTable } from './tables/backgroundJobsTable';
+import { BranchesTable } from './tables/branchesTable';
+import { CommitsTable } from './tables/commitsTable';
+import { CommitBranchesTable } from './tables/commitBranchesTable';
+import { DebugReportsTable } from './tables/debugReportsTable';
+import { EntitiesTable } from './tables/entitiesTable';
+import { EntityBranchesTable } from './tables/entityBranchesTable';
+import { FilesTable } from './tables/filesTable';
+import { ProjectsTable } from './tables/projectsTable';
+import { ScenariosTable } from './tables/scenariosTable';
+import { ScenarioCommentsTable } from './tables/scenarioCommentsTable';
+import { UsersTable } from './tables/usersTable';
+import { GithubUsersTable } from './tables/githubUsersTable';
+import { StatementsTable } from './tables/statementsTable';
+
+/**
+ * A module for the 'select' perspective on our database tables, but with
+ * relations (e.g. from joins) mixed in.
+ */
+
+export type DbAnalysisBranch = Selectable<AnalysisBranchesTable> & {
+  analysis?: DbAnalysis;
+  branch?: DbBranch;
+  entity?: DbEntity;
+};
+
+export type DbAnalysis = Selectable<AnalysesTable> & {
+  entity?: DbEntity;
+  file?: DbFile;
+  commit?: DbCommit;
+  project?: DbProject;
+  scenarios?: DbScenario[];
+  analysis_branches?: DbAnalysisBranch[];
+};
+
+export type DbBackgroundJob = Selectable<BackgroundJobsTable>;
+
+export type DbBranch = Selectable<BranchesTable> & {
+  commits?: DbCommit[];
+  analysis_branches?: DbAnalysisBranch[];
+};
+
+export type DbCommit = Selectable<CommitsTable> & {
+  author?: DbCommitAuthor;
+  branch?: DbBranch;
+  background_jobs?: DbBackgroundJob[];
+  mergedBranch?: DbBranch;
+  analyses?: DbAnalysis[];
+  entities?: DbEntity[];
+  commit_branches?: DbCommitBranch[];
+};
+
+// virtual schema for joined author data (see loadCommit.ts)
+export type DbCommitAuthor = {
+  username: string;
+  avatarUrl: string;
+  preferredUsername?: string;
+};
+
+export type DbCommitBranch = Selectable<CommitBranchesTable> & {
+  commit?: DbCommit;
+  branch?: DbBranch;
+};
+
+export type DbEntityBranch = Selectable<EntityBranchesTable> & {
+  entity?: DbEntity;
+};
+
+export type DbEntity = Selectable<EntitiesTable> & {
+  analyses?: DbAnalysis[];
+  commit?: DbCommit;
+  entity_branches?: DbEntityBranch[];
+};
+
+export type DbFile = Selectable<FilesTable>;
+
+export type DbProject = Selectable<ProjectsTable> & {
+  branches?: Selectable<BranchesTable>[];
+  files?: Selectable<FilesTable>[];
+};
+
+export type DbScenarioComment = Selectable<ScenarioCommentsTable> & {
+  user?: DbUser;
+};
+
+export type DbScenario = Selectable<ScenariosTable> & {
+  analysis?: DbAnalysis;
+  user_scenarios?: DbUserScenario[];
+  scenario_comments?: DbScenarioComment[];
+};
+
+export type DbGithubUser = Selectable<GithubUsersTable>;
+
+export type DbUser = Selectable<UsersTable> & {
+  github_user?: DbGithubUser;
+};
+
+export type DbUserScenario = Selectable<UserScenariosTable> & {
+  user?: DbUser;
+};
+
+export type DbStatement = Selectable<StatementsTable>;
+
+export type DbDebugReport = Selectable<DebugReportsTable>;

package/analyzer-template/packages/database/src/lib/kysely/tables/debugReportsTable.ts

@@ -0,0 +1,61 @@
+import { Kysely, ColumnDefinitionBuilder } from 'kysely';
+import type { Generated, JSONColumnType } from 'kysely';
+
+import { schemaField, defaultNow } from '../schemaHelpers';
+
+interface DebugReportMetadata {
+  timestamp: string;
+  git: {
+    sha: string;
+    branch: string;
+    isDirty: boolean;
+    remoteUrl: string | null;
+  };
+  versions: {
+    cli: string;
+    webserver: object | null;
+    node: string;
+  };
+  system: {
+    platform: string;
+    arch: string;
+  };
+}
+
+const debugReportsSchema = {
+  id: schemaField<Generated<string>>(),
+  project_slug: schemaField<string>(),
+  s3_key: schemaField<string>(),
+  file_size_bytes: schemaField<number | null>(),
+  metadata: schemaField<JSONColumnType<DebugReportMetadata> | null>(),
+  status: schemaField<string | null>(),
+  created_at: schemaField<Generated<string>>(),
+  uploaded_at: schemaField<string | null>(),
+} as const;
+
+export type DebugReportsTable = {
+  [K in keyof typeof debugReportsSchema]: (typeof debugReportsSchema)[K];
+};
+
+export const DebugReportsTableColumns = Object.keys(
+  debugReportsSchema,
+) as (keyof DebugReportsTable)[];
+
+export async function createDebugReportsTable(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .createTable('debug_reports')
+    .addColumn('id', 'uuid', (col: ColumnDefinitionBuilder) => col.primaryKey())
+    .addColumn('project_slug', 'varchar', (col: ColumnDefinitionBuilder) =>
+      col.notNull(),
+    )
+    .addColumn('s3_key', 'varchar', (col: ColumnDefinitionBuilder) =>
+      col.notNull(),
+    )
+    .addColumn('file_size_bytes', 'bigint')
+    .addColumn('metadata', 'text') // JSON stored as text in SQLite
+    .addColumn('status', 'varchar')
+    .addColumn('created_at', 'datetime', defaultNow(true))
+    .addColumn('uploaded_at', 'datetime')
+    .ifNotExists()
+    .execute();
+}

package/analyzer-template/packages/database/src/lib/loadCommit.ts

@@ -0,0 +1,162 @@
+import { Database, getDatabase, getJsonHelper } from './kysely/db';
+import dbToCommit from './dbToCommit';
+import {
+  DbAnalysis,
+  DbBranch,
+  DbCommit,
+  DbScenario,
+} from './kysely/tableRelations';
+import { awsLog } from '~codeyam/utils';
+import { ExpressionBuilder } from 'kysely';
+import { ScenariosTableColumns } from './kysely/tables/scenariosTable';
+
+interface LoadCommitArgs {
+  id?: string;
+  projectId?: string;
+  sha?: string;
+  branchId?: string;
+}
+
+async function loadBranchForCommit(branchId: string): Promise<DbBranch | null> {
+  const db = getDatabase();
+
+  try {
+    return (
+      (await db
+        .selectFrom('branches')
+        .selectAll()
+        .where('id', '=', branchId)
+        .executeTakeFirst()) || null
+    );
+  } catch (error) {
+    awsLog('CodeYam Error: Loading branch for commit', error, { branchId });
+    return null;
+  }
+}
+
+async function loadAnalysesForCommit(commitId: string): Promise<DbAnalysis[]> {
+  const db = getDatabase();
+
+  const { jsonArrayFrom } = getJsonHelper();
+
+  try {
+    const analyses = await db
+      .selectFrom('analyses')
+      .selectAll('analyses')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        jsonArrayFrom(
+          eb
+            .selectFrom('scenarios')
+            .select(ScenariosTableColumns)
+            .whereRef('scenarios.analysis_id', '=', 'analyses.id')
+            .$castTo<DbScenario>(),
+        ).as('scenarios'),
+      )
+      .where('commit_id', '=', commitId)
+      .execute();
+
+    return analyses;
+  } catch (error) {
+    awsLog('CodeYam Error: Loading analyses for commit', error, { commitId });
+    return [];
+  }
+}
+
+export default async function loadCommit({
+  id,
+  projectId,
+  sha,
+  branchId,
+}: LoadCommitArgs) {
+  const db = getDatabase();
+
+  try {
+    let query = db
+      .selectFrom('commits')
+      .selectAll('commits')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getCommitAuthorsAsJson(eb).as('author'),
+      );
+
+    if (id) {
+      query = query.where('id', '=', id);
+    }
+
+    if (sha) {
+      query = query.where('sha', '=', sha);
+    }
+
+    if (projectId) {
+      query = query.where('project_id', '=', projectId);
+    }
+
+    if (branchId) {
+      query = query.where('branch_id', '=', branchId);
+    }
+
+    query = query.orderBy('committed_at', 'desc').limit(1);
+
+    const commit = await query.executeTakeFirst();
+
+    if (!commit) {
+      awsLog('CodeYam Error: Commit not found', null, {
+        id,
+        projectId,
+        sha,
+        branchId,
+      });
+      return null;
+    }
+
+    // Load branch and mergedBranch separately
+    const [branch, mergedBranch, analyses] = await Promise.all([
+      commit.branch_id
+        ? loadBranchForCommit(commit.branch_id)
+        : Promise.resolve(null),
+      commit.merged_branch_id
+        ? loadBranchForCommit(commit.merged_branch_id)
+        : Promise.resolve(null),
+      loadAnalysesForCommit(commit.id),
+    ]);
+
+    // Construct the full DbCommit object
+    const dbCommit: DbCommit = {
+      ...commit,
+      branch: branch || undefined,
+      mergedBranch: mergedBranch || undefined,
+      analyses,
+    };
+
+    return dbToCommit(dbCommit);
+  } catch (error) {
+    awsLog('CodeYam Error: Database error loading commit', error, {
+      id,
+      projectId,
+      sha,
+      branchId,
+    });
+    return null;
+  }
+}
+
+// Export for reuse in other loaders
+export function getCommitAuthorsAsJson(
+  eb: ExpressionBuilder<Database, 'commits'>,
+) {
+  const { jsonObjectFrom } = getJsonHelper();
+
+  return jsonObjectFrom(
+    eb
+      .selectFrom('github_users')
+      .select([
+        'username',
+        'preferred_username as preferredUsername',
+        'avatar_url as avatarUrl',
+      ])
+      .where(
+        'github_users.username',
+        '=',
+        eb.ref('commits.author_github_username'),
+      ),
+  );
+}

package/analyzer-template/packages/database/src/lib/loadEntities.ts

@@ -0,0 +1,94 @@
+import { Entity } from '~codeyam/types';
+import dbToEntity from './dbToEntity';
+import { getDatabase } from './kysely/db';
+
+export interface LoadEntitiesArgs {
+  projectId?: string;
+  branchId?: string;
+  fileIds?: string[];
+  filePaths?: string[];
+  names?: string[];
+  shas?: string[];
+}
+
+export default async function loadEntities({
+  projectId,
+  branchId,
+  fileIds,
+  filePaths,
+  names,
+  shas,
+}: LoadEntitiesArgs): Promise<Entity[] | null> {
+  if (
+    (fileIds && fileIds.length == 0) ||
+    (filePaths && filePaths.length == 0) ||
+    (names && names.length == 0) ||
+    (shas && shas.length == 0)
+  ) {
+    return [];
+  }
+
+  if (shas && shas.length > 50) {
+    const allEntities: Entity[] = [];
+    for (let i = 0; i < shas.length; i += 50) {
+      const chunk = shas.slice(i, i + 50);
+      const chunkEntities = await loadEntities({
+        projectId,
+        branchId,
+        fileIds,
+        filePaths,
+        names,
+        shas: chunk,
+      });
+
+      if (chunkEntities) {
+        allEntities.push(...chunkEntities);
+      }
+    }
+    return allEntities;
+  }
+
+  const db = getDatabase();
+
+  try {
+    const query = db
+      .selectFrom('entities')
+      .selectAll('entities')
+      .$if(!!branchId, (qb) =>
+        qb
+          .innerJoin(
+            'entity_branches',
+            'entity_branches.entity_sha',
+            'entities.sha',
+          )
+          .where('entity_branches.branch_id', '=', branchId),
+      )
+      .$if(!!projectId, (qb) => qb.where('entities.project_id', '=', projectId))
+      .$if(!!shas, (qb) => qb.where('entities.sha', 'in', shas))
+      .$if(!!filePaths, (qb) => qb.where('entities.file_path', 'in', filePaths))
+      .$if(!!names, (qb) => qb.where('entities.name', 'in', names))
+      .$if(!!fileIds, (qb) => qb.where('entities.file_id', 'in', fileIds));
+
+    const entities = await query.execute();
+
+    if (!entities || entities.length === 0) {
+      console.log('Load Entities: No entities found', {
+        projectId,
+        fileIds,
+        filePaths,
+        shas,
+      });
+      return null;
+    }
+
+    return entities.map(dbToEntity);
+  } catch (error) {
+    console.log('Load Entities: Error occurred', error, {
+      projectId,
+      fileIds,
+      filePaths,
+      shas,
+    });
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadEntity.ts

@@ -0,0 +1,105 @@
+import { ExpressionBuilder } from 'kysely';
+import { getDatabase } from './kysely/db';
+import dbToEntity from './dbToEntity';
+import { awsLog } from '~codeyam/utils';
+import { getEntityBranchesAsJson } from './loadEntityBranches';
+
+export interface loadEntityArgs {
+  projectId: string;
+  sha: string;
+}
+
+export interface loadEntityByNameAndPathArgs {
+  projectId: string;
+  name: string;
+  filePath: string;
+}
+
+export default async function loadEntity({ projectId, sha }: loadEntityArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.sha', '=', sha)
+      .executeTakeFirst();
+
+    if (!entity) {
+      // Only log at debug level - Entity not found is expected in E2E baseline mode
+      // where entities are created fresh and don't exist in the database yet
+      if (process.env.CODEYAM_E2E_BASELINE_MODE !== 'true') {
+        awsLog('CodeYam Error: Load Entity: Entity not found', null, {
+          projectId,
+          sha,
+        });
+      }
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      sha,
+    });
+    return null;
+  }
+}
+
+/**
+ * Load entity by name and file path instead of SHA.
+ * This is useful for E2E tests where we want to avoid hardcoding SHAs.
+ */
+export async function loadEntityByNameAndPath({
+  projectId,
+  name,
+  filePath,
+}: loadEntityByNameAndPathArgs) {
+  const db = getDatabase();
+
+  try {
+    const entity = await db
+      .selectFrom('entities')
+      .innerJoin('files', 'entities.file_id', 'files.id')
+      .selectAll('entities')
+      .select((eb: ExpressionBuilder<any, any>) =>
+        getEntityBranchesAsJson(eb, (seb) =>
+          seb.whereRef('entity_branches.entity_sha', '=', 'entities.sha'),
+        ).as('entity_branches'),
+      )
+      .where('files.project_id', '=', projectId)
+      .where('entities.name', '=', name)
+      .where('files.path', '=', filePath)
+      .executeTakeFirst();
+
+    if (!entity) {
+      awsLog(
+        'CodeYam Error: Load Entity: Entity not found by name and path',
+        null,
+        {
+          projectId,
+          name,
+          filePath,
+        },
+      );
+      return null;
+    }
+
+    return dbToEntity(entity);
+  } catch (error) {
+    awsLog('CodeYam Error: Load Entity: Database error', error, {
+      projectId,
+      name,
+      filePath,
+    });
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFile.ts

@@ -0,0 +1,42 @@
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+
+export default async function loadFile({
+  projectId,
+  id,
+  path,
+}: {
+  projectId?: string;
+  id?: string;
+  path?: string;
+}) {
+  const db = getDatabase();
+
+  let query = db.selectFrom('files').selectAll();
+
+  if (projectId) {
+    query = query.where('project_id', '=', projectId);
+  }
+
+  if (id) {
+    query = query.where('id', '=', id);
+  }
+
+  if (path) {
+    query = query.where('path', '=', path);
+  }
+
+  try {
+    const file = await query.executeTakeFirst();
+
+    if (!file) {
+      console.log('Error loading file', { projectId, id, path }, 'Not found');
+      return null;
+    }
+
+    return dbToFile(file);
+  } catch (error) {
+    console.log('Error loading file', { projectId, id, path }, error);
+    return null;
+  }
+}

package/analyzer-template/packages/database/src/lib/loadFiles.ts

@@ -0,0 +1,134 @@
+import { File } from '~codeyam/types';
+import dbToFile from './dbToFile';
+import { getDatabase } from './kysely/db';
+import { Selectable } from 'kysely';
+import { FilesTable } from './kysely/tables/filesTable';
+
+const RANGE_LIMIT = 1000;
+
+/**
+ * Load only file paths for a project (lightweight version)
+ * Use this when you only need paths, not full file objects (~80% memory savings)
+ */
+export async function loadFilePaths(projectId: string): Promise<string[]> {
+  const db = getDatabase();
+  const paths: string[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      const query = db
+        .selectFrom('files')
+        .select('path') // Only select path column
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      paths.push(...filesData.map((f) => f.path));
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return paths;
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading file paths in loadFilePaths',
+      error,
+    );
+    return [];
+  }
+}
+
+interface LoadFilesArgs {
+  projectId: string;
+  filePaths?: string[];
+  fileIds?: string[];
+  fileNames?: string[];
+}
+
+export default async function loadFiles({
+  projectId,
+  filePaths,
+  fileIds,
+  fileNames,
+}: LoadFilesArgs): Promise<File[] | null> {
+  if (filePaths && filePaths.length > 50) {
+    const allFiles: File[] = [];
+    for (let i = 0; i < filePaths.length; i += 50) {
+      const chunk = filePaths.slice(i, i + 50);
+      const chunkFiles = await loadFiles({
+        projectId,
+        filePaths: chunk,
+        fileIds,
+        fileNames,
+      });
+
+      if (chunkFiles) {
+        allFiles.push(...chunkFiles);
+      }
+    }
+    return allFiles;
+  }
+
+  const db = getDatabase();
+  const files: Selectable<FilesTable>[] = [];
+  let offset = 0;
+
+  try {
+    while (true) {
+      let query = db
+        .selectFrom('files')
+        .selectAll()
+        .where('project_id', '=', projectId)
+        .limit(RANGE_LIMIT)
+        .offset(offset);
+
+      if (filePaths) {
+        if (filePaths.length === 0) return [];
+        query = query.where('path', 'in', filePaths);
+      }
+
+      if (fileIds) {
+        if (fileIds.length === 0) return [];
+        query = query.where('id', 'in', fileIds);
+      }
+
+      if (fileNames) {
+        if (fileNames.length === 0) return [];
+        query = query.where('name', 'in', fileNames);
+      }
+
+      const filesData = await query.execute();
+
+      if (!filesData || filesData.length === 0) {
+        break;
+      }
+
+      files.push(...filesData);
+
+      if (filesData.length < RANGE_LIMIT) {
+        break;
+      } else {
+        offset += RANGE_LIMIT;
+      }
+    }
+
+    return files?.map(dbToFile);
+  } catch (error) {
+    console.log(
+      'CodeYam Error: Error loading project files in loadFiles',
+      error,
+    );
+    return null;
+  }
+}