git-dlp 0.2.0

package/src/getPathContents/PathContentsMetaInfo.ts

@@ -0,0 +1,85 @@
+import * as Effect from 'effect/Effect'
+
+import { CastToReadableStream } from '../castToReadableStream.ts'
+import { ParsedMetaInfoAboutPathContentsFromGitHubAPI } from './ParsedMetaInfoAboutPathContentsFromGitHubAPI.ts'
+import { parseGitLFSObjectEither } from './parseGitLFSObjectEither.ts'
+
+export const PathContentsMetaInfo = Effect.gen(function* () {
+  const response = yield* ParsedMetaInfoAboutPathContentsFromGitHubAPI
+
+  const { type, name, path, size } = response
+
+  if (type === 'dir') {
+    const { entries, sha: treeSha } = response
+
+    if (name && path)
+      return {
+        type,
+        name,
+        path,
+        treeSha,
+        entries,
+        meta: 'This nested directory can be downloaded as a git tree',
+      } as const
+
+    return {
+      type,
+      treeSha,
+      entries,
+      meta: 'This root directory of the repo can be downloaded as a git tree',
+    } as const
+  }
+
+  // This is quite forgiving implementation. I had the choice to throw
+  // errors whenever GitHub's API didn't follow its documentation. I even
+  // did that initially, but when I looked at the resulting mess, I changed
+  // my mind not to. For example, I could throw errors in the following
+  // cases:
+
+  // 1. When the size is between 1 MB and 100 MB per documentation, I
+  //    should never receive data. Instead, I should receive an empty
+  //    "content" field and an "encoding" field equal to "none." If this
+  //    promise was broken, I could have thrown an error. Instead, if I
+  //    receive a 50 MB file with the correct encoding, I will parse and
+  //    return it.
+  // 2. Per the documentation, all files larger than 100 MB should be put
+  //    into Git LFS storage. If I receive a 110 MB file inlined, I'll not
+  //    fail; I'll parse and return it.
+  // 3. Per documentation when size less than 1MB it MUST be inlined. If it
+  //    wasn't inlined I could have thrown an error, but instead, I just
+  //    returned an object representing the message "It's a blob, download
+  //    it elsewhere"
+  // 4. Per documentation files larger than 100 mb must be in a git LFS
+  //    storage and it's assumed that git LFS annotation will be provided.
+  //    But if it's not provided, instead of throwing an error, I returned
+  //    an object representing the message "It's a blob, download it
+  //    elsewhere"
+
+  // In the end it leads to much lower complexity with a ton of IFs removed
+
+  const { content, encoding, sha: blobSha, ...restFileObjectFields } = response
+  const base = { ...restFileObjectFields, blobSha }
+
+  if (encoding === 'none')
+    return { ...base, meta: 'This file can be downloaded as a blob' } as const
+
+  const contentAsBuffer = Buffer.from(content, encoding)
+
+  const potentialGitLFSObject = yield* parseGitLFSObjectEither({
+    contentAsBuffer,
+    expectedContentSize: size,
+  })
+
+  if (typeof potentialGitLFSObject === 'object')
+    return {
+      ...base,
+      ...potentialGitLFSObject,
+      meta: 'This file can be downloaded as a git-LFS object',
+    } as const
+
+  return {
+    ...base,
+    contentStream: CastToReadableStream(Effect.succeed(contentAsBuffer)),
+    meta: 'This file is small enough that GitHub API decided to inline it',
+  } as const
+})

package/src/getPathContents/RawStreamOfRepoPathContentsFromGitHubAPI.ts

@@ -0,0 +1,6 @@
+import { CastToReadableStream } from '../castToReadableStream.ts'
+import { RepoPathContentsFromGitHubAPI } from './RepoPathContentsFromGitHubAPI.ts'
+
+export const RawStreamOfRepoPathContentsFromGitHubAPI = CastToReadableStream(
+  RepoPathContentsFromGitHubAPI('raw', true),
+)

package/src/getPathContents/RepoPathContentsFromGitHubAPI.ts

@@ -0,0 +1,82 @@
+import { RequestError } from '@octokit/request-error'
+import type { OctokitResponse } from '@octokit/types'
+
+import * as Cause from 'effect/Cause'
+import * as Effect from 'effect/Effect'
+
+import {
+  GitHubApiNoCommitFoundForGitRefError,
+  GitHubApiRepoIsEmptyError,
+  GitHubApiThingNotExistsOrYouDontHaveAccessError,
+  parseCommonGitHubApiErrors,
+} from '../commonErrors.ts'
+import { InputConfigTag } from '../configContext.ts'
+import { OctokitTag } from '../octokit.ts'
+
+// TODO make better typesignature so that it actually reflects what's returned:
+// stream or parsed body use params as a generic. (but generally fuck github API
+// and their fucking octokit)
+export const RepoPathContentsFromGitHubAPI = Effect.fn(
+  'getRepoPathContentsFromGitHubAPI',
+)(function* (format: 'object' | 'raw', streamBody?: boolean) {
+  const octokit = yield* OctokitTag
+
+  const { gitRef, pathToEntityInRepo, repo } = yield* InputConfigTag
+  // TODO: improve sitation on what's default with git ref. Better set null
+  // instead of empty string? to signify it's not present actually. Also should
+  // note that HEAD could mean something different when ref is not specified.
+  // Should consider if we should fallback to HEAD or just not specifiying the
+  // property at all.
+
+  return yield* Effect.tryPromise({
+    try: signal =>
+      octokit.request('GET /repos/{owner}/{repo}/contents/{path}', {
+        owner: repo.owner,
+        repo: repo.name,
+        path: pathToEntityInRepo,
+        ...(gitRef && { ref: gitRef }),
+        request: {
+          signal,
+          parseSuccessResponseBody: !streamBody,
+        },
+        mediaType: { format },
+        headers: {
+          'X-GitHub-Api-Version': '2022-11-28',
+        },
+      }),
+    catch: error => {
+      if (!(error instanceof RequestError))
+        return new Cause.UnknownException(
+          error,
+          'Failed to request contents at the path inside GitHub repo',
+        )
+
+      const potentialErrorMessage = (error.response as ResponseWithError)?.data
+        ?.message
+
+      if (error.status === 404 && potentialErrorMessage)
+        return parseNotFoundErrors(potentialErrorMessage, error, gitRef)
+
+      return parseCommonGitHubApiErrors(error)
+    },
+  })
+})
+
+const parseNotFoundErrors = (
+  potentialErrorMessage: string,
+  error: RequestError,
+  gitRef: string,
+) => {
+  if (potentialErrorMessage === 'This repository is empty.')
+    return new GitHubApiRepoIsEmptyError(error)
+
+  if (gitRef && potentialErrorMessage.startsWith('No commit found for the ref'))
+    return new GitHubApiNoCommitFoundForGitRefError(error, { gitRef })
+
+  return new GitHubApiThingNotExistsOrYouDontHaveAccessError(error)
+}
+
+type ResponseWithError = OctokitResponse<
+  { message?: string } | undefined,
+  number
+>

package/src/getPathContents/index.ts

@@ -0,0 +1,7 @@
+export { FailedToParseResponseFromRepoPathContentsMetaInfoAPIError } from './ParsedMetaInfoAboutPathContentsFromGitHubAPI.ts'
+export { PathContentsMetaInfo } from './PathContentsMetaInfo.ts'
+export {
+  FailedToParseGitLFSInfoError,
+  InconsistentExpectedAndRealContentSizeError,
+} from './parseGitLFSObjectEither.ts'
+export { RawStreamOfRepoPathContentsFromGitHubAPI } from './RawStreamOfRepoPathContentsFromGitHubAPI.ts'

package/src/getPathContents/parseGitLFSObjectEither.ts

@@ -0,0 +1,143 @@
+import { outdent } from 'outdent'
+
+import * as Either from 'effect/Either'
+import * as ParseResult from 'effect/ParseResult'
+import * as Schema from 'effect/Schema'
+
+import {
+  buildTaggedErrorClassVerifyingCause,
+  type TaggedErrorClass,
+} from '../TaggedErrorVerifyingCause.ts'
+
+export const parseGitLFSObjectEither = ({
+  contentAsBuffer,
+  expectedContentSize,
+}: {
+  contentAsBuffer: Buffer<ArrayBuffer>
+  expectedContentSize: number
+}) =>
+  Either.gen(function* () {
+    // gitLFS info usually is no longer than MAX_GIT_LFS_INFO_SIZE bytes
+    const contentAsString = contentAsBuffer
+      .subarray(0, MAX_GIT_LFS_INFO_SIZE)
+      .toString('utf8')
+
+    const parsingResult = Either.mapLeft(
+      decodeGitLFSInfoSchema(contentAsString.match(gitLFSInfoRegexp)?.groups),
+      cause =>
+        new FailedToParseGitLFSInfoError(cause, {
+          partOfContentThatCouldBeGitLFSInfo: contentAsString,
+        }),
+    )
+
+    const matchedByRegexpAndParsedByEffectSchema = Either.isRight(parsingResult)
+    const doesSizeFromGitLFSInfoAlignWithExpectedContentSize =
+      Either.isRight(parsingResult) &&
+      parsingResult.right.size === expectedContentSize
+
+    const shouldFailIfItIsNotGitLFS =
+      contentAsBuffer.byteLength !== expectedContentSize
+
+    const isThisAGitLFSObject =
+      matchedByRegexpAndParsedByEffectSchema &&
+      doesSizeFromGitLFSInfoAlignWithExpectedContentSize
+
+    if (isThisAGitLFSObject)
+      return {
+        gitLFSObjectIdSha256: parsingResult.right.oidSha256,
+        gitLFSVersion: parsingResult.right.version,
+      } as const
+
+    if (shouldFailIfItIsNotGitLFS)
+      return yield* Either.left(
+        new InconsistentExpectedAndRealContentSizeError({
+          actual: contentAsBuffer.byteLength,
+          expected: expectedContentSize,
+          gitLFSInfo: parsingResult,
+        }),
+      )
+
+    return 'This is not a git LFS object' as const
+  })
+
+// there are some responses that look like
+// `version https://git-lfs.github.com/spec/v1
+// oid sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+// size 128
+// `
+// and the only variable thing in it is the size at the end, and I assume
+// that supported file size is not greater than 100 GB
+const MAX_GIT_LFS_INFO_SIZE = 137
+// Don't add regexp /g modifier, it breaks match groups
+const gitLFSInfoRegexp =
+  /^version (?<version>https:\/\/git-lfs\.github\.com\/spec\/v1)\noid sha256:(?<oidSha256>[0-9a-f]{64})\nsize (?<size>[1-9]\d{0,11})\n$/m
+
+const GitLFSInfoSchema = Schema.Struct({
+  version: Schema.NonEmptyTrimmedString,
+  oidSha256: Schema.NonEmptyTrimmedString,
+  size: Schema.NumberFromString,
+})
+
+const decodeGitLFSInfoSchema = Schema.decodeUnknownEither(GitLFSInfoSchema, {
+  exact: true,
+})
+
+// Extracting to a separate type is required by JSR, so that consumers of the
+// library will have much faster type inference
+export type FailedToParseGitLFSInfoErrorClass = TaggedErrorClass<{
+  ErrorName: 'FailedToParseGitLFSInfoError'
+  ExpectedCauseClass: typeof ParseResult.ParseError
+  DynamicContext: { partOfContentThatCouldBeGitLFSInfo: string }
+}>
+
+const _1: FailedToParseGitLFSInfoErrorClass =
+  buildTaggedErrorClassVerifyingCause<{
+    partOfContentThatCouldBeGitLFSInfo: string
+  }>()(
+    'FailedToParseGitLFSInfoError',
+    `Failed to parse git LFS announcement`,
+    ParseResult.ParseError,
+  )
+
+export class FailedToParseGitLFSInfoError extends _1 {}
+
+type InconsistentSizesDynamicContext = {
+  actual: number
+  expected: number
+  gitLFSInfo: Either.Either<
+    Readonly<{
+      version: string
+      oidSha256: string
+      size: number
+    }>,
+    InstanceType<FailedToParseGitLFSInfoErrorClass>
+  >
+}
+
+// Extracting to a separate type is required by JSR, so that consumers of the
+// library will have much faster type inference
+
+const _2: TaggedErrorClass<{
+  ErrorName: 'InconsistentExpectedAndRealContentSizeError'
+  StaticContext: { comment: string }
+  DynamicContext: InconsistentSizesDynamicContext
+}> = buildTaggedErrorClassVerifyingCause<InconsistentSizesDynamicContext>()(
+  'InconsistentExpectedAndRealContentSizeError',
+  ctx =>
+    `Got file with size ${ctx.actual} bytes while expecting ${ctx.expected} bytes`,
+  void 0,
+  {
+    comment: outdent({ newline: ' ' })`
+      If we weren't successful in parsing it as git LFS object
+      announcement using RegExp and Effect.Schema, we just do a basic size
+      consistency check. The check implements the second marker of it
+      being a Git LFS object as a backup to checking does "content" look
+      like a Git LFS object. If GitHub API's "size" field is different
+      from actual size of "content" field, it means either our schema with
+      regexp fucked up, or GitHub API did. If it doesn't throw, it means
+      there's no reason to assume it's a Git LFS object.
+    `,
+  },
+)
+
+export class InconsistentExpectedAndRealContentSizeError extends _2 {}

package/src/getReadableTarGzStreamOfRepoDirectory.ts

@@ -0,0 +1,65 @@
+import { RequestError } from '@octokit/request-error'
+
+import * as Cause from 'effect/Cause'
+import * as Effect from 'effect/Effect'
+import * as EFunction from 'effect/Function'
+
+import { CastToReadableStream } from './castToReadableStream.ts'
+import {
+  GitHubApiGeneralUserError,
+  parseCommonGitHubApiErrors,
+} from './commonErrors.ts'
+import { InputConfigTag } from './configContext.ts'
+import { OctokitTag } from './octokit.ts'
+
+export const getReadableTarGzStreamOfRepoDirectory = (
+  gitRefWhichWillBeUsedToIdentifyGitTree?: string,
+) =>
+  EFunction.pipe(
+    requestTarballFromGitHubAPI(gitRefWhichWillBeUsedToIdentifyGitTree),
+    Effect.map(({ data }) => data),
+    CastToReadableStream,
+  )
+
+// TODO: better return type signature so that we actually show what's returned
+// buffer, stream or something else. Octokit sucks
+const requestTarballFromGitHubAPI = (
+  gitRefWhichWillBeUsedToIdentifyGitTree = '',
+) =>
+  Effect.gen(function* () {
+    const octokit = yield* OctokitTag
+
+    const {
+      repo: { owner, name },
+    } = yield* InputConfigTag
+
+    return yield* Effect.tryPromise({
+      try: signal =>
+        octokit.request('GET /repos/{owner}/{repo}/tarball/{ref}', {
+          owner,
+          repo: name,
+          ref: gitRefWhichWillBeUsedToIdentifyGitTree,
+          request: {
+            signal,
+            parseSuccessResponseBody: false,
+          },
+          headers: {
+            'X-GitHub-Api-Version': '2022-11-28',
+          },
+        }),
+      catch: error => {
+        if (!(error instanceof RequestError))
+          return new Cause.UnknownException(
+            error,
+            'Failed to request .tar.gz file from GitHub API',
+          )
+
+        if (error.status === 400)
+          return new GitHubApiGeneralUserError(error, {
+            notes: 'Error happened probably because you asked for empty repo',
+          })
+
+        return parseCommonGitHubApiErrors(error)
+      },
+    })
+  })

package/src/index.ts

@@ -0,0 +1,13 @@
+/**
+ * @module
+ */
+
+export * from './cli.ts'
+export type {
+  InputConfig,
+  OutputConfig,
+  SingleTargetConfig,
+} from './configContext.ts'
+export { downloadEntityFromRepo } from './downloadEntityFromRepo.ts'
+export * from './errors.ts'
+export { OctokitLayer } from './octokit.ts'

package/src/octokit.ts

@@ -0,0 +1,15 @@
+import { Octokit, type OctokitOptions } from '@octokit/core'
+
+import * as Context from 'effect/Context'
+import * as Layer from 'effect/Layer'
+
+// Extracting to a separate type is required by JSR, so that consumers of the
+// library will have much faster type inference
+type OctokitTag = Context.Tag<Octokit, Octokit>
+
+export const OctokitTag: OctokitTag = Context.GenericTag<Octokit>('OctokitTag')
+
+export const OctokitLayer: (
+  options?: OctokitOptions,
+) => Layer.Layer<Octokit, never, never> = (options?: OctokitOptions) =>
+  Layer.succeed(OctokitTag, OctokitTag.of(new Octokit(options)))

package/src/unpackRepoFolderTarGzStreamToFs.ts

@@ -0,0 +1,61 @@
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { createGunzip } from 'node:zlib'
+
+import { extract } from 'tar-fs'
+
+import * as Effect from 'effect/Effect'
+
+import { OutputConfigTag } from './configContext.ts'
+import {
+  buildTaggedErrorClassVerifyingCause,
+  type TaggedErrorClass,
+} from './TaggedErrorVerifyingCause.ts'
+
+// TODO: Use this maybe for tar.gz unpacking?
+// https://github.com/leonitousconforti/eftar
+// TODO: Or implement my own wrapper around node:zlib?
+
+export const unpackRepoFolderTarGzStreamToFs = <E, R>(
+  self: Effect.Effect<Readable, E, R>,
+) =>
+  Effect.gen(function* () {
+    const tarGzStream = yield* self
+
+    const {
+      localPathAtWhichEntityFromRepoWillBeAvailable:
+        pathToLocalDirWhichWillHaveContentsOfRepoDir,
+    } = yield* OutputConfigTag
+
+    yield* Effect.tryPromise({
+      try: signal =>
+        pipeline(
+          tarGzStream,
+          createGunzip(),
+          extract(pathToLocalDirWhichWillHaveContentsOfRepoDir, {
+            map: header => {
+              // GitHub creates archive with nested dir inside which has all
+              // the files we need, so we remove this dir's name from the
+              // beginning
+              header.name = header.name.replace(/^[^/]*\/(.*)/, '$1')
+              return header
+            },
+          }),
+          { signal },
+        ),
+      catch: cause =>
+        new FailedToUnpackRepoFolderTarGzStreamToFsError({ cause }),
+    })
+  })
+
+// Extracting to a separate type is required by JSR, so that consumers of the
+// library will have much faster type inference
+const _1: TaggedErrorClass<{
+  ErrorName: 'FailedToUnpackRepoFolderTarGzStreamToFsError'
+  DynamicContext: { cause: unknown }
+}> = buildTaggedErrorClassVerifyingCause<{ cause: unknown }>()(
+  'FailedToUnpackRepoFolderTarGzStreamToFsError',
+  'Error: Failed to unpack to fs received from GitHub .tar.gz stream of repo folder contents',
+)
+
+export class FailedToUnpackRepoFolderTarGzStreamToFsError extends _1 {}

package/src/writeFileStreamToDestinationPath.ts

@@ -0,0 +1,45 @@
+import { createWriteStream } from 'node:fs'
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+
+import * as Effect from 'effect/Effect'
+
+import { OutputConfigTag } from './configContext.ts'
+import {
+  buildTaggedErrorClassVerifyingCause,
+  type TaggedErrorClass,
+} from './TaggedErrorVerifyingCause.ts'
+
+// TODO: use effect platform streams
+
+export const writeFileStreamToDestinationPath = <E, R>(
+  self: Effect.Effect<Readable, E, R>,
+) =>
+  Effect.gen(function* () {
+    const fileStream = yield* self
+
+    const {
+      localPathAtWhichEntityFromRepoWillBeAvailable: localDownloadedFilePath,
+    } = yield* OutputConfigTag
+
+    yield* Effect.tryPromise({
+      try: signal =>
+        pipeline(fileStream, createWriteStream(localDownloadedFilePath), {
+          signal,
+        }),
+      catch: cause =>
+        new FailedToWriteFileStreamToDestinationPathError({ cause }),
+    })
+  })
+
+// Extracting to a separate type is required by JSR, so that consumers of the
+// library will have much faster type inference
+const _1: TaggedErrorClass<{
+  ErrorName: 'FailedToWriteFileStreamToDestinationPathError'
+  DynamicContext: { cause: unknown }
+}> = buildTaggedErrorClassVerifyingCause<{ cause: unknown }>()(
+  'FailedToWriteFileStreamToDestinationPathError',
+  'Error: Failed to write file stream to destination path',
+)
+
+export class FailedToWriteFileStreamToDestinationPathError extends _1 {}

package/template.env

@@ -0,0 +1,13 @@
+# Get here: https://github.com/settings/tokens/new?description=Read%20repo%20contents%20access%20to%20fetch-github-folder&scopes=public_repo&default_expires_at=none
+GITHUB_ACCESS_TOKEN='ghp_1234567890abcdefghij'
+
+REPO_OWNER='apache'
+
+REPO_NAME='superset'
+
+PATH_TO_ENTITY_IN_REPO='docker'
+
+# If commented, default branch in repo will be used
+# GIT_REF='main'
+
+DESTINATION_PATH='/tmp/tmp.pPuwKB2gSZ/docker'