@commitspark/git-adapter-github 0.21.0 → 0.30.0

package/src/github-api/get-file-paths.ts

@@ -0,0 +1,66 @@
+import { GitHubRepositoryOptions } from '../index'
+import { AxiosCacheInstance } from 'axios-cache-interceptor'
+import { ErrorCode, GitAdapterError } from '@commitspark/git-adapter'
+import { getPathEntryFolder } from '../util/path-factory'
+import { handleHttpErrors } from '../errors'
+import { GITHUB_REST_API_URL } from '../types'
+
+export const getFilePaths = async (
+  gitRepositoryOptions: GitHubRepositoryOptions,
+  axiosCacheInstance: AxiosCacheInstance,
+  treeSha: string,
+): Promise<string[]> => {
+  const pathEntryFolder = getPathEntryFolder(gitRepositoryOptions)
+  const { repositoryOwner, repositoryName, accessToken } = gitRepositoryOptions
+
+  // hard limit of 100,000 entries and 7MB response size; see https://docs.github.com/en/rest/git/trees?apiVersion=2022-11-28#get-a-tree
+  const restApiUrl =
+    GITHUB_REST_API_URL +
+    `/repos/${repositoryOwner}/${repositoryName}/git/trees/${treeSha}?recursive=1`
+
+  let response
+  try {
+    response = await axiosCacheInstance.get(restApiUrl, {
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        Accept: 'application/vnd.github+json',
+      },
+      timeout: 30000,
+    })
+  } catch (error) {
+    handleHttpErrors(error)
+  }
+
+  if (response === undefined) {
+    throw new GitAdapterError(
+      ErrorCode.INTERNAL_ERROR,
+      'Failed to fetch repository file list.',
+    )
+  }
+
+  const data = response.data as GitHubTreeResponse
+
+  if (data.truncated) {
+    throw new GitAdapterError(
+      ErrorCode.INTERNAL_ERROR,
+      `Too many files in repository.`,
+    )
+  }
+
+  return data.tree
+    .filter(
+      (entry) =>
+        entry.type === 'blob' && entry.path.startsWith(pathEntryFolder),
+    )
+    .map((entry) => entry.path)
+}
+
+interface GitHubTreeResponse {
+  tree: GitHubTreeEntry[]
+  truncated: boolean
+}
+
+interface GitHubTreeEntry {
+  path: string
+  type: 'blob' | 'tree' | 'commit'
+}

package/src/{util → github-api}/graphql-query-factory.ts

@@ -1,18 +1,10 @@
-export function createBlobsContentQuery(): string {
+export function createSingleBlobContentQuery(): string {
   return `
       query ($repositoryOwner: String!, $repositoryName: String!, $expression: String!) { 
         repository(owner: $repositoryOwner, name: $repositoryName) {
           object(expression: $expression) {
-            ... on Tree {
-              entries {
-                name
-                object {
-                  __typename
-                  ... on Blob {
-                    text
-                  }
-                }
-              }
+            ... on Blob {
+              text
             }
           }
         }
@@ -20,18 +12,47 @@ export function createBlobsContentQuery(): string {
     `
 }
 
-export function createBlobContentQuery(): string {
-  return `
-      query ($repositoryOwner: String!, $repositoryName: String!, $expression: String!) { 
-        repository(owner: $repositoryOwner, name: $repositoryName) {
-          object(expression: $expression) {
-            ... on Blob {
-              text
-            }
-          }
+export function createBlobsContentByFilePathsQuery(
+  filePaths: string[],
+  commitHash: string,
+  batchSize: number,
+): { queries: string[]; queryFilenameAliasMap: Map<string, string> } {
+  const queries = []
+  const queryFilenameAliasMap = new Map<string, string>()
+  for (
+    let fileIndex = 0;
+    fileIndex < filePaths.length;
+    fileIndex += batchSize
+  ) {
+    // cut file content query into batches and use GraphQL aliases for maximum throughput in regard to GitHub API limits
+
+    const batchFilenames = filePaths.slice(fileIndex, fileIndex + batchSize)
+
+    let query = `
+      query ($repositoryOwner: String!, $repositoryName: String!) { 
+        repository(owner: $repositoryOwner, name: $repositoryName) {`
+
+    for (const [j, filename] of batchFilenames.entries()) {
+      const fileAliasIndex = fileIndex + j
+      const fileAlias = `file${fileAliasIndex}`
+      // using one query variable per file would significantly blow up query size, which would in turn require a lower
+      // batch size to avoid GitHub erroring out, and in consequence much lower throughput; we therefore use this poor
+      // man's manual escaping of filenames instead and write them directly into the query strings
+      const escapedExpression = JSON.stringify(`${commitHash}:${filename}`)
+      query += `    ${fileAlias}: object(expression: ${escapedExpression}) {
+      ... on Blob {
+          text
         }
       }
-    `
+`
+      queryFilenameAliasMap.set(fileAlias, filename)
+    }
+    query += `  }
+}`
+    queries.push(query)
+  }
+
+  return { queries, queryFilenameAliasMap }
 }
 
 export function createCommitMutation(): string {

package/src/types.ts

@@ -0,0 +1,6 @@
+export const ENTRY_EXTENSION = '.yaml'
+export const PATH_SCHEMA_FILE = 'commitspark/schema/schema.graphql'
+export const PATH_ENTRY_FOLDER = 'commitspark/entries/'
+export const QUERY_BATCH_SIZE = 250
+export const GITHUB_GRAPHQL_API_URL = 'https://api.github.com/graphql'
+export const GITHUB_REST_API_URL = 'https://api.github.com'

package/src/util/entries-to-actions-converter.ts

@@ -2,7 +2,7 @@ import { EntryDraft } from '@commitspark/git-adapter'
 import { stringify } from 'yaml'
 import { AdditionModel } from '../model/addition.model'
 import { DeletionModel } from '../model/deletion.model'
-import { ENTRY_EXTENSION } from './types'
+import { ENTRY_EXTENSION } from '../types'
 
 export function convertEntriesToActions(
   entryDrafts: EntryDraft[],

package/src/util/entry-factory.ts

@@ -1,29 +1,20 @@
 import { parse } from 'yaml'
 import { Entry } from '@commitspark/git-adapter'
-import { ENTRY_EXTENSION } from './types'
+import { ENTRY_EXTENSION } from '../types'
+import { getPathEntryFolder } from './path-factory'
+import { GitHubRepositoryOptions } from '../index'
 
-interface TreeEntry {
-  name: string
-  object: {
-    __typename: string
-    text: string
-  }
-}
-
-export function createEntriesFromBlobsQueryResponseData(
-  entries: TreeEntry[],
+export function createEntriesFromFileContent(
+  gitRepositoryOptions: GitHubRepositoryOptions,
+  filePathContentMap: Map<string, string>,
 ): Entry[] {
-  return entries
-    .filter(
-      (entry: TreeEntry) =>
-        entry.name.endsWith(ENTRY_EXTENSION) &&
-        entry.object['__typename'] === 'Blob',
-    )
-    .map((entry: TreeEntry) => {
-      const fileContent = parse(entry.object.text)
-      const id = entry.name.substring(
-        0,
-        entry.name.length - ENTRY_EXTENSION.length,
+  return Array.from(filePathContentMap)
+    .filter(([filePath]) => filePath.endsWith(ENTRY_EXTENSION))
+    .map(([filePath, content]) => {
+      const fileContent = parse(content)
+      const id = filePath.substring(
+        getPathEntryFolder(gitRepositoryOptions).length, // strip folder path back out
+        filePath.length - ENTRY_EXTENSION.length,
       )
       return {
         id: id,

package/src/util/path-factory.ts

@@ -1,5 +1,5 @@
 import { GitHubRepositoryOptions } from '../index'
-import { PATH_ENTRY_FOLDER, PATH_SCHEMA_FILE } from './types'
+import { PATH_ENTRY_FOLDER, PATH_SCHEMA_FILE } from '../types'
 
 export function getPathSchema(gitRepositoryOptions: GitHubRepositoryOptions) {
   return gitRepositoryOptions.pathSchemaFile ?? PATH_SCHEMA_FILE
@@ -11,8 +11,8 @@ export function getPathEntryFolder(
   const pathEntryFolder =
     gitRepositoryOptions.pathEntryFolder ?? PATH_ENTRY_FOLDER
 
-  if (pathEntryFolder.endsWith('/')) {
-    return pathEntryFolder.substring(0, pathEntryFolder.length - 1)
+  if (!pathEntryFolder.endsWith('/')) {
+    return pathEntryFolder + '/'
   }
 
   return pathEntryFolder

package/dist/cjs/util/graphql-query-factory.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"graphql-query-factory.js","sourceRoot":"","sources":["../../../src/util/graphql-query-factory.ts"],"names":[],"mappings":";;AAAA,0DAoBC;AAED,wDAYC;AAED,oDA+BC;AAED,0DAeC;AApFD,SAAgB,uBAAuB;IACrC,OAAO;;;;;;;;;;;;;;;;;;KAkBJ,CAAA;AACL,CAAC;AAED,SAAgB,sBAAsB;IACpC,OAAO;;;;;;;;;;KAUJ,CAAA;AACL,CAAC;AAED,SAAgB,oBAAoB;IAClC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA6BJ,CAAA;AACL,CAAC;AAED,SAAgB,uBAAuB;IACrC,OAAO;;;;;;;;;;;;;KAaJ,CAAA;AACL,CAAC"}

package/dist/cjs/util/types.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PATH_ENTRY_FOLDER = exports.PATH_SCHEMA_FILE = exports.ENTRY_EXTENSION = void 0;
-exports.ENTRY_EXTENSION = '.yaml';
-exports.PATH_SCHEMA_FILE = 'commitspark/schema/schema.graphql';
-exports.PATH_ENTRY_FOLDER = 'commitspark/entries/';
-//# sourceMappingURL=types.js.map

package/dist/cjs/util/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/util/types.ts"],"names":[],"mappings":";;;AAAa,QAAA,eAAe,GAAG,OAAO,CAAA;AACzB,QAAA,gBAAgB,GAAG,mCAAmC,CAAA;AACtD,QAAA,iBAAiB,GAAG,sBAAsB,CAAA"}

package/dist/esm/util/graphql-query-factory.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"graphql-query-factory.js","sourceRoot":"","sources":["../../../src/util/graphql-query-factory.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,uBAAuB;IACrC,OAAO;;;;;;;;;;;;;;;;;;KAkBJ,CAAA;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,OAAO;;;;;;;;;;KAUJ,CAAA;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA6BJ,CAAA;AACL,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,OAAO;;;;;;;;;;;;;KAaJ,CAAA;AACL,CAAC"}

package/dist/esm/util/types.js

@@ -1,4 +0,0 @@
-export const ENTRY_EXTENSION = '.yaml';
-export const PATH_SCHEMA_FILE = 'commitspark/schema/schema.graphql';
-export const PATH_ENTRY_FOLDER = 'commitspark/entries/';
-//# sourceMappingURL=types.js.map

package/dist/esm/util/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/util/types.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAA;AACtC,MAAM,CAAC,MAAM,gBAAgB,GAAG,mCAAmC,CAAA;AACnE,MAAM,CAAC,MAAM,iBAAiB,GAAG,sBAAsB,CAAA"}

package/dist/types/util/graphql-query-factory.d.ts

@@ -1,5 +0,0 @@
-export declare function createBlobsContentQuery(): string;
-export declare function createBlobContentQuery(): string;
-export declare function createCommitMutation(): string;
-export declare function createLatestCommitQuery(): string;
-//# sourceMappingURL=graphql-query-factory.d.ts.map

package/dist/types/util/graphql-query-factory.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"graphql-query-factory.d.ts","sourceRoot":"","sources":["../../../src/util/graphql-query-factory.ts"],"names":[],"mappings":"AAAA,wBAAgB,uBAAuB,IAAI,MAAM,CAoBhD;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAY/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CA+B7C;AAED,wBAAgB,uBAAuB,IAAI,MAAM,CAehD"}

package/dist/types/util/types.d.ts

@@ -1,4 +0,0 @@
-export declare const ENTRY_EXTENSION = ".yaml";
-export declare const PATH_SCHEMA_FILE = "commitspark/schema/schema.graphql";
-export declare const PATH_ENTRY_FOLDER = "commitspark/entries/";
-//# sourceMappingURL=types.d.ts.map

package/dist/types/util/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/util/types.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,eAAe,UAAU,CAAA;AACtC,eAAO,MAAM,gBAAgB,sCAAsC,CAAA;AACnE,eAAO,MAAM,iBAAiB,yBAAyB,CAAA"}

package/src/util/types.ts

@@ -1,3 +0,0 @@
-export const ENTRY_EXTENSION = '.yaml'
-export const PATH_SCHEMA_FILE = 'commitspark/schema/schema.graphql'
-export const PATH_ENTRY_FOLDER = 'commitspark/entries/'