@codeyam/codeyam-cli 0.1.0-staging.ad31e3e → 0.1.0-staging.ae0de75

package/analyzer-template/packages/aws/src/lib/ecs/ecsTaskFactory.ts

@@ -5,51 +5,22 @@ export type EcsStartConfig = {
 };
 
 /**
- * Prepare to run an ECS task for the classical analysis-then-capture flow
- * which uses a 'project' image for analysis and a 'playwright' image for capture.
+ * Prepare to run an ECS task for analysis with distributed capture orchestration.
+ * Uses the unified 'combo' image which includes both analysis and capture capabilities.
  */
-export function getEcsStartConfigForAnalysisThenCapture(
-  projectEnvironmentVariables: Record<string, string>,
-  playwrightEnvironmentVariables: Record<string, string>,
-): EcsStartConfig {
-  return {
-    taskDefinition: getEcsTaskDefinitionName(
-      EcsTaskDefinitionUse.AnalyzeThenCapture,
-    ),
-    containerEnvironmentVariables: [
-      [
-        getEcsImageNameForType(EcsImageType.Project),
-        projectEnvironmentVariables,
-      ],
-      [
-        getEcsImageNameForType(EcsImageType.Playwright),
-        playwrightEnvironmentVariables,
-      ],
-    ],
-  };
-}
-
-/**
- * Prepare to run an ECS task for the new analysis flow with distributed capture
- * tasks orchestrated from only a 'project' image.
- */
-export function getEcsStartConfigFoAnalysisWithOrchestration(
+export function getEcsStartConfigForAnalysis(
   environmentVariables: Record<string, string>,
 ): EcsStartConfig {
   return {
-    taskDefinition: getEcsTaskDefinitionName(
-      EcsTaskDefinitionUse.AnalyzeWithOrchestration,
-    ),
-    containerEnvironmentVariables: [
-      [getEcsImageNameForType(EcsImageType.Project), environmentVariables],
-    ],
+    taskDefinition: getEcsTaskDefinitionName(EcsTaskDefinitionUse.Analysis),
+    containerEnvironmentVariables: [[getEcsImageName(), environmentVariables]],
   };
 }
 
 /**
  * Prepare to run an ECS worker task that implements the capture portion of the
- * distributed orchestration, which uses a 'combo' image. This task is launched
- * from a task created with `getEcsStartConfigFoAnalysisWithOrchestration()`
+ * distributed orchestration. Uses the same unified 'combo' image.
+ * This task is launched from a task created with `getEcsStartConfigForAnalysis()`
  */
 export function getEcsStartConfigForOrchestrationTask(
   environmentVariables: Record<string, string>,
@@ -58,52 +29,29 @@ export function getEcsStartConfigForOrchestrationTask(
     taskDefinition: getEcsTaskDefinitionName(
       EcsTaskDefinitionUse.OrchestrationTask,
     ),
-    containerEnvironmentVariables: [
-      [getEcsImageNameForType(EcsImageType.Combo), environmentVariables],
-    ],
+    containerEnvironmentVariables: [[getEcsImageName(), environmentVariables]],
   };
 }
 
 export enum EcsTaskDefinitionUse {
-  /* A 'project' image runs analysis while a 'playwright' image captures the entities. */
-  AnalyzeThenCapture = 'analyze-then-capture',
-  /* A 'project' image runs analysis while firing up distributed capture orchestration. */
-  AnalyzeWithOrchestration = 'analyze-with-orchestration',
-  /* A worker task that combines 'project' and 'playwright' code, launched by orchestration. */
+  /* Main analysis task - runs analysis and fires up distributed capture orchestration. */
+  Analysis = 'analysis',
+  /* A worker task launched by orchestration to capture scenarios. */
   OrchestrationTask = 'orchestration-task',
 }
 
 export function getEcsTaskDefinitionName(forUse: EcsTaskDefinitionUse): string {
   switch (forUse) {
-    case EcsTaskDefinitionUse.AnalyzeThenCapture:
-      return `${process.env.ECR_REPOSITORY_NAME}-project-and-playwright`;
-    case EcsTaskDefinitionUse.AnalyzeWithOrchestration:
-      return `${process.env.ECR_REPOSITORY_NAME}-project`;
+    case EcsTaskDefinitionUse.Analysis:
+      return `${process.env.ECR_REPOSITORY_NAME}-analysis`;
     case EcsTaskDefinitionUse.OrchestrationTask:
-      return `${process.env.ECR_REPOSITORY_NAME}-combo`;
+      return `${process.env.ECR_REPOSITORY_NAME}-orchestration`;
     default:
       throw new Error(`Unknown task definition use: ${forUse}`);
   }
 }
 
-export enum EcsImageType {
-  /* The virtualized server that runs the project code. */
-  Project = 'project',
-  /* The playwright capture process which waits for the project to be ready. */
-  Playwright = 'playwright',
-  /* A combo image that combines both project and playwright code, used by orchestration. */
-  Combo = 'combo',
-}
-
-export function getEcsImageNameForType(type: EcsImageType): string {
-  switch (type) {
-    case EcsImageType.Project:
-      return `${process.env.ECR_REPOSITORY_NAME}-project`;
-    case EcsImageType.Playwright:
-      return `${process.env.ECR_REPOSITORY_NAME}-playwright`;
-    case EcsImageType.Combo:
-      return `${process.env.ECR_REPOSITORY_NAME}-combo`;
-    default:
-      throw new Error(`Unknown image type: ${type}`);
-  }
+/** Returns the name of the unified Docker image used for all ECS tasks. */
+export function getEcsImageName(): string {
+  return `${process.env.ECR_REPOSITORY_NAME}-combo`;
 }

package/analyzer-template/packages/aws/src/lib/s3/checkS3ObjectExists.ts

@@ -0,0 +1,47 @@
+import { S3Client, HeadObjectCommand } from '@aws-sdk/client-s3';
+
+interface CheckS3ObjectExistsArgs {
+  bucketName: string;
+  key: string;
+}
+
+interface CheckS3ObjectExistsResult {
+  exists: boolean;
+  size?: number;
+}
+
+/**
+ * Check if an object exists in S3 using a HEAD request.
+ * Returns the object size if it exists.
+ */
+export default async function checkS3ObjectExists({
+  bucketName,
+  key,
+}: CheckS3ObjectExistsArgs): Promise<CheckS3ObjectExistsResult> {
+  const s3Client = new S3Client({});
+
+  const command = new HeadObjectCommand({
+    Bucket: bucketName,
+    Key: key,
+  });
+
+  try {
+    const response = await s3Client.send(command);
+    return {
+      exists: true,
+      size: response.ContentLength,
+    };
+  } catch (error: unknown) {
+    // NotFound is expected when object doesn't exist
+    if (
+      error &&
+      typeof error === 'object' &&
+      'name' in error &&
+      error.name === 'NotFound'
+    ) {
+      return { exists: false };
+    }
+    // Re-throw unexpected errors
+    throw error;
+  }
+}

package/analyzer-template/packages/aws/src/lib/s3/getPresignedUrl.ts

@@ -25,7 +25,7 @@ const DEFAULT_DOWNLOAD_EXPIRY = 24 * 60 * 60; // 24 hours
  * Generate a presigned URL for uploading a file to S3.
  * The client can PUT directly to this URL without AWS credentials.
  */
-export async function getPresignedUploadUrl({
+export function getPresignedUploadUrl({
   bucketName,
   key,
   contentType = 'application/gzip',
@@ -46,7 +46,7 @@ export async function getPresignedUploadUrl({
  * Generate a presigned URL for downloading a file from S3.
  * Anyone with this URL can GET the file without AWS credentials.
  */
-export async function getPresignedDownloadUrl({
+export function getPresignedDownloadUrl({
   bucketName,
   key,
   expiresInSeconds = DEFAULT_DOWNLOAD_EXPIRY,

package/analyzer-template/packages/aws/src/lib/s3/uploadFileToS3.ts

@@ -24,11 +24,18 @@ export default async function uploadFileToS3({
   });
 
   try {
+    let lastLoggedMB = 0;
     upload.on('httpUploadProgress', (progress) => {
-      console.log(progress);
+      const loadedMB = Math.floor((progress.loaded ?? 0) / (1024 * 1024));
+      // Log every 10MB to avoid spam
+      if (loadedMB >= lastLoggedMB + 10) {
+        lastLoggedMB = loadedMB;
+        console.log(`Uploading ${key}... ${loadedMB}MB`);
+      }
     });
 
     await upload.done();
+    console.log(`Uploaded ${key}`);
   } catch (error) {
     console.log('Error. File not uploaded.', error);
     throw error;

package/analyzer-template/packages/database/client.ts

@@ -0,0 +1,35 @@
+/**
+ * This index file re-exports modules which are safe to use in a browser
+ * environment with our current set of polyfills. Two considerations:
+ * - It must compile: it can't rely on node modules we don't polyfill
+ * - It must use Supabase for its REST capabilities. None of the db.ts
+ *   infrastructure must be invoked in this file or any of its imports.
+ *
+ *   Kysely itself is fine.
+ */
+
+export { getSupabase, getAdminSupabase } from './src/lib/supabase';
+
+export type { SupabaseClient } from '@supabase/supabase-js';
+export { default as upsertFiles_Client } from './src/lib/client/upsertFiles_Client';
+export { default as loadAnalyses_Client } from './src/lib/client/loadAnalysesInClient';
+export { default as loadBranches_Client } from './src/lib/client/loadBranches_Client';
+export { default as loadAnalysis_Client } from './src/lib/client/loadAnalysis_Client';
+
+export { default as listenForCommits_Client } from './src/lib/client/listenForCommits_Client';
+
+export { default as dbToAnalysis } from './src/lib/dbToAnalysis';
+export { default as dbToBranch } from './src/lib/dbToBranch';
+export { default as dbToCommit } from './src/lib/dbToCommit';
+export { default as dbToEntity } from './src/lib/dbToEntity';
+export { default as dbToFile } from './src/lib/dbToFile';
+export { default as dbToProject } from './src/lib/dbToProject';
+export { default as dbToScenario } from './src/lib/dbToScenario';
+export { default as dbToUserScenario } from './src/lib/dbToUserScenario';
+export { default as dbToScenarioComment } from './src/lib/dbToScenarioComment';
+
+// NOTE: generateSha is NOT exported here because it uses Node's crypto module
+// which is server-only. Import from '~codeyam/database' (server) instead.
+
+// Export ONLY types from tableRelations (no runtime code that imports Kysely)
+export type * from './src/lib/kysely/tableRelationsTypes';

package/analyzer-template/packages/database/index.ts

@@ -0,0 +1,94 @@
+export { getSupabase, getAdminSupabase } from './src/lib/supabase';
+export { default as createOrUpdateBranchCommitStats } from './src/lib/createOrUpdateBranchCommitStats';
+export { default as createProject } from './src/lib/createProject';
+export { default as dbToAnalysis } from './src/lib/dbToAnalysis';
+export { default as dbToAnalysisBranch } from './src/lib/dbToAnalysisBranch';
+export { default as dbToBranch } from './src/lib/dbToBranch';
+export { default as dbToCommit } from './src/lib/dbToCommit';
+export { default as dbToEntity } from './src/lib/dbToEntity';
+export { default as dbToFile } from './src/lib/dbToFile';
+export { default as dbToProject } from './src/lib/dbToProject';
+export { default as dbToScenario } from './src/lib/dbToScenario';
+export { default as dbToUserScenario } from './src/lib/dbToUserScenario';
+export { default as deleteBranch } from './src/lib/deleteBranch';
+export { default as deleteFile } from './src/lib/deleteFile';
+export { default as deleteEntities } from './src/lib/deleteEntities';
+export { default as deleteScenarios } from './src/lib/deleteScenarios';
+export { default as generateSha } from './src/lib/generateSha';
+export { default as loadAnalyses } from './src/lib/loadAnalyses';
+export { default as loadAnalysis } from './src/lib/loadAnalysis';
+export { default as loadAnalysisBranches } from './src/lib/loadAnalysisBranches';
+export { default as loadBackgroundJob } from './src/lib/loadBackgroundJob';
+export { default as loadBranch } from './src/lib/loadBranch';
+export { default as loadBranches } from './src/lib/loadBranches';
+export { default as loadCommit } from './src/lib/loadCommit';
+export { default as loadCommitBranches } from './src/lib/loadCommitBranches';
+export { default as loadCommitMetadata } from './src/lib/loadCommitMetadata';
+export { default as loadCommits } from './src/lib/loadCommits';
+export { default as loadEntities } from './src/lib/loadEntities';
+export {
+  default as loadEntity,
+  loadEntityByNameAndPath,
+} from './src/lib/loadEntity';
+export { default as loadEntityBranches } from './src/lib/loadEntityBranches';
+export { default as loadFile } from './src/lib/loadFile';
+export { default as loadFiles, loadFilePaths } from './src/lib/loadFiles';
+export { default as loadMostRecentPreviousAnalysis } from './src/lib/loadMostRecentPreviousAnalysis';
+export { default as loadProject } from './src/lib/loadProject';
+export { default as loadReadyToBeCapturedAnalyses } from './src/lib/loadReadyToBeCapturedAnalyses';
+export { default as loadScenario } from './src/lib/loadScenario';
+export { default as loadStatement } from './src/lib/loadStatement';
+export { default as saveBackgroundEvent } from './src/lib/saveBackgroundEvent';
+export { default as saveEntityStatements } from './src/lib/saveEntityStatements';
+export { default as saveFiles } from './src/lib/saveFiles';
+export { default as saveStatement } from './src/lib/saveStatement';
+export { default as updateBackgroundJobProgress } from './src/lib/updateBackgroundJobProgress';
+export { default as updateCommitMetadata } from './src/lib/updateCommitMetadata';
+export { default as updateEntityBranch } from './src/lib/updateEntityBranch';
+export { default as updateFreshAnalysisMetadata } from './src/lib/updateFreshAnalysisMetadata';
+export { default as updateFreshAnalysisStatus } from './src/lib/updateFreshAnalysisStatus';
+export { default as updateFreshAnalysisStatusWithScenarios } from './src/lib/updateFreshAnalysisStatusWithScenarios';
+export { default as updateProjectMetadata } from './src/lib/updateProjectMetadata';
+export { default as upsertAnalyses } from './src/lib/upsertAnalyses';
+export { default as upsertAnalysesWithScenarios } from './src/lib/upsertAnalysesWithScenarios';
+export { default as upsertAnalysisBranches } from './src/lib/upsertAnalysisBranches';
+export { default as upsertBackgroundJob } from './src/lib/upsertBackgroundJob';
+export { default as upsertBranches } from './src/lib/upsertBranches';
+export { default as upsertCommitBranches } from './src/lib/upsertCommitBranches';
+export { default as upsertCommits } from './src/lib/upsertCommits';
+export { default as upsertEntities } from './src/lib/upsertEntities';
+export { default as upsertEntityBranches } from './src/lib/upsertEntityBranches';
+export { default as upsertFiles } from './src/lib/upsertFiles';
+export { default as upsertGithubUser } from './src/lib/upsertGithubUser';
+export { default as upsertProjects } from './src/lib/upsertProjects';
+export { default as upsertScenarios } from './src/lib/upsertScenarios';
+
+export * from './src/lib/kysely/db';
+
+export { type Insertable } from 'kysely';
+
+export * from './src/lib/kysely/tableRelations';
+
+export * from './src/lib/kysely/tables/analysesTable';
+export * from './src/lib/kysely/tables/analysisBranchesTable';
+export * from './src/lib/kysely/tables/backgroundJobsTable';
+export * from './src/lib/kysely/tables/branchesTable';
+export * from './src/lib/kysely/tables/commitBranchesTable';
+export * from './src/lib/kysely/tables/commitsTable';
+export * from './src/lib/kysely/tables/editorScenariosTable';
+export * from './src/lib/kysely/tables/entitiesTable';
+export * from './src/lib/kysely/tables/entityBranchesTable';
+export * from './src/lib/kysely/tables/entityStatementsTable';
+export * from './src/lib/kysely/tables/filesTable';
+export * from './src/lib/kysely/tables/githubPayloadsTable';
+export * from './src/lib/kysely/tables/githubUsersTable';
+export * from './src/lib/kysely/tables/projectsTable';
+export * from './src/lib/kysely/tables/scenarioCommentsTable';
+export * from './src/lib/kysely/tables/scenariosTable';
+export * from './src/lib/kysely/tables/statementsTable';
+export * from './src/lib/kysely/tables/teamsTable';
+export * from './src/lib/kysely/tables/userScenariosTable';
+export * from './src/lib/kysely/tables/usersTable';
+export * from './src/lib/kysely/tables/userTeamsTable';
+
+export type { SupabaseClient } from '@supabase/supabase-js';

package/analyzer-template/packages/database/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@codeyam/supabase",
+  "version": "1.0.0",
+  "description": "",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "test": "jest --watchAll --verbose"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@supabase/supabase-js": "2.88.0",
+    "better-sqlite3": "^12.6.2",
+    "fetch-retry": "^6.0.0",
+    "kysely": "^0.28.11",
+    "pg": "^8.19.0"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/jest": "^30.0.0",
+    "@types/pg": "^8.18.0",
+    "@types/ws": "^8.18.1"
+  }
+}

package/analyzer-template/packages/database/src/lib/analysisBranchToDb.ts

@@ -0,0 +1,28 @@
+import { AnalysisBranch } from '~codeyam/types';
+import { DbAnalysisBranch } from './kysely/tableRelations';
+const randomUUID = () => crypto.randomUUID();
+import { AnalysisBranchesTable } from './kysely/tables/analysisBranchesTable';
+import { type Insertable } from 'kysely';
+
+export default function analysisBranchToDb(
+  analysisBranch: AnalysisBranch,
+): Insertable<AnalysisBranchesTable> {
+  const { id, analysisId, entitySha, branchId, ...remaining } = analysisBranch;
+
+  delete remaining.analysis;
+  delete remaining.entity;
+  delete remaining.branch;
+
+  if ('createdAt' in remaining) {
+    (remaining as unknown as DbAnalysisBranch).created_at = remaining.createdAt;
+    delete remaining.createdAt;
+  }
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    analysis_id: analysisId,
+    entity_sha: entitySha,
+    branch_id: branchId,
+  };
+}

package/analyzer-template/packages/database/src/lib/analysisToDb.ts

@@ -0,0 +1,59 @@
+import { Analysis } from '~codeyam/types';
+import { type Insertable } from 'kysely';
+import { AnalysesTable } from './kysely/tables/analysesTable';
+const randomUUID = () => crypto.randomUUID();
+
+export default function analysisToDb(
+  analysis: Analysis,
+): Insertable<AnalysesTable> {
+  const {
+    id,
+    projectId,
+    commitId,
+    fileId,
+    filePath,
+    entitySha,
+    entityType,
+    entityName,
+    previousAnalysisId,
+    entity,
+    dependencyAnalyzedTreeSha,
+    analyzedTreeSha,
+    branchCommitSha,
+    committedAt,
+    completedAt,
+    createdAt,
+    updatedAt,
+    metadata,
+    status,
+    commit,
+    file,
+    project,
+    scenarios,
+    analysisBranches,
+    branches,
+    ...remaining
+  } = analysis;
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    metadata: metadata ? JSON.stringify(metadata) : undefined,
+    status: status ? JSON.stringify(status) : undefined,
+    project_id: projectId,
+    commit_id: commitId,
+    file_id: fileId,
+    file_path: filePath,
+    entity_sha: entitySha,
+    entity_type: entityType,
+    entity_name: entityName,
+    previous_analysis_id: previousAnalysisId,
+    dependency_analyzed_tree_sha: dependencyAnalyzedTreeSha,
+    analyzed_tree_sha: analyzedTreeSha,
+    branch_commit_sha: branchCommitSha,
+    committed_at: committedAt,
+    completed_at: completedAt,
+    created_at: createdAt,
+    updated_at: updatedAt,
+  };
+}

package/analyzer-template/packages/database/src/lib/branchToDb.ts

@@ -0,0 +1,24 @@
+import { Branch } from '~codeyam/types';
+import { type Insertable } from 'kysely';
+import { BranchesTable } from './kysely/tables/branchesTable';
+const randomUUID = () => crypto.randomUUID();
+
+export default function branchToDb(branch: Branch): Insertable<BranchesTable> {
+  const { id, projectId, activeAt, contentChangedAt, metadata, ...remaining } =
+    branch;
+
+  delete remaining.commits;
+  delete remaining.files;
+  delete remaining.analyses;
+  delete remaining.createdAt;
+  delete remaining.updatedAt;
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    project_id: projectId,
+    active_at: activeAt,
+    content_changed_at: contentChangedAt,
+    metadata: metadata ? JSON.stringify(metadata) : null,
+  };
+}

package/analyzer-template/packages/database/src/lib/commitBranchToDb.ts

@@ -0,0 +1,20 @@
+import { CommitBranch } from '~codeyam/types';
+import { type Insertable } from 'kysely';
+const randomUUID = () => crypto.randomUUID();
+import { CommitBranchesTable } from './kysely/tables/commitBranchesTable';
+
+export default function commitBranchToDb(
+  commitBranch: CommitBranch,
+): Insertable<CommitBranchesTable> {
+  const { id, commitId, branchId, ...remaining } = commitBranch;
+
+  delete remaining.commit;
+  delete remaining.branch;
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    commit_id: commitId,
+    branch_id: branchId,
+  };
+}

package/analyzer-template/packages/database/src/lib/commitToDb.ts

@@ -0,0 +1,47 @@
+import { Commit } from '~codeyam/types';
+import { CommitsTable } from './kysely/tables/commitsTable';
+import { type Insertable } from 'kysely';
+const randomUUID = () => crypto.randomUUID();
+
+export default function commitToDb(
+  commit: Commit,
+  projectId: number | string,
+): Insertable<CommitsTable> {
+  const {
+    id,
+    projectId: _projectId,
+    branchId,
+    mergedBranchId,
+    aiMessage,
+    htmlUrl,
+    analyzedAt,
+    committedAt,
+    author,
+    metadata,
+    files,
+    ...remaining
+  } = commit;
+
+  delete remaining.branch;
+  delete remaining.mergedBranch;
+  delete remaining.backgroundJob;
+  delete remaining.analyses;
+  delete remaining.parents;
+  delete remaining.entities;
+  delete remaining.commitBranches;
+
+  return {
+    ...remaining,
+    id: id ?? randomUUID(),
+    project_id: _projectId ?? String(projectId),
+    metadata: metadata ? JSON.stringify(metadata) : undefined,
+    files: files ? JSON.stringify(files) : undefined,
+    branch_id: branchId,
+    merged_branch_id: mergedBranchId,
+    author_github_username: author?.username,
+    html_url: htmlUrl,
+    ai_message: aiMessage,
+    analyzed_at: analyzedAt,
+    committed_at: committedAt,
+  };
+}

package/analyzer-template/packages/database/src/lib/fileToDb.ts

@@ -0,0 +1,17 @@
+import { File } from '~codeyam/types';
+import { type Insertable } from 'kysely';
+import { FilesTable } from './kysely/tables/filesTable';
+const randomUUID = () => crypto.randomUUID();
+
+export default function fileToDb(file: File): Insertable<FilesTable> {
+  return {
+    id: file.id ?? randomUUID(),
+    project_id: file.projectId!,
+    name: file.name,
+    path: file.path,
+    deleted: file.deleted ?? null,
+    metadata: file.metadata ? JSON.stringify(file.metadata) : null,
+    created_at: file.createdAt ?? new Date().toISOString(),
+    updated_at: file.updatedAt ?? null,
+  };
+}