git-dlp 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +290 -0
- package/dist/git-dl.js +43 -0
- package/dist/git-dl.js.map +1 -0
- package/dist/package.json +138 -0
- package/dist/src/TaggedErrorVerifyingCause.js +27 -0
- package/dist/src/TaggedErrorVerifyingCause.js.map +1 -0
- package/dist/src/castToReadableStream.js +28 -0
- package/dist/src/castToReadableStream.js.map +1 -0
- package/dist/src/cli.js +8 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/commandLineParams.js +172 -0
- package/dist/src/commandLineParams.js.map +1 -0
- package/dist/src/commonErrors.js +60 -0
- package/dist/src/commonErrors.js.map +1 -0
- package/dist/src/configContext.js +12 -0
- package/dist/src/configContext.js.map +1 -0
- package/dist/src/downloadEntityFromRepo.js +21 -0
- package/dist/src/downloadEntityFromRepo.js.map +1 -0
- package/dist/src/errors.js +10 -0
- package/dist/src/errors.js.map +1 -0
- package/dist/src/getPathContents/ParsedMetaInfoAboutPathContentsFromGitHubAPI.js +43 -0
- package/dist/src/getPathContents/ParsedMetaInfoAboutPathContentsFromGitHubAPI.js.map +1 -0
- package/dist/src/getPathContents/PathContentsMetaInfo.js +71 -0
- package/dist/src/getPathContents/PathContentsMetaInfo.js.map +1 -0
- package/dist/src/getPathContents/RawStreamOfRepoPathContentsFromGitHubAPI.js +4 -0
- package/dist/src/getPathContents/RawStreamOfRepoPathContentsFromGitHubAPI.js.map +1 -0
- package/dist/src/getPathContents/RepoPathContentsFromGitHubAPI.js +51 -0
- package/dist/src/getPathContents/RepoPathContentsFromGitHubAPI.js.map +1 -0
- package/dist/src/getPathContents/index.js +5 -0
- package/dist/src/getPathContents/index.js.map +1 -0
- package/dist/src/getPathContents/parseGitLFSObjectEither.js +70 -0
- package/dist/src/getPathContents/parseGitLFSObjectEither.js.map +1 -0
- package/dist/src/getReadableTarGzStreamOfRepoDirectory.js +39 -0
- package/dist/src/getReadableTarGzStreamOfRepoDirectory.js.map +1 -0
- package/dist/src/index.js +8 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/octokit.js +6 -0
- package/dist/src/octokit.js.map +1 -0
- package/dist/src/unpackRepoFolderTarGzStreamToFs.js +31 -0
- package/dist/src/unpackRepoFolderTarGzStreamToFs.js.map +1 -0
- package/dist/src/writeFileStreamToDestinationPath.js +22 -0
- package/dist/src/writeFileStreamToDestinationPath.js.map +1 -0
- package/dist-types/git-dl.d.ts +3 -0
- package/dist-types/git-dl.d.ts.map +1 -0
- package/dist-types/src/TaggedErrorVerifyingCause.d.ts +38 -0
- package/dist-types/src/TaggedErrorVerifyingCause.d.ts.map +1 -0
- package/dist-types/src/castToReadableStream.d.ts +11 -0
- package/dist-types/src/castToReadableStream.d.ts.map +1 -0
- package/dist-types/src/cli.d.ts +8 -0
- package/dist-types/src/cli.d.ts.map +1 -0
- package/dist-types/src/commandLineParams.d.ts +108 -0
- package/dist-types/src/commandLineParams.d.ts.map +1 -0
- package/dist-types/src/commonErrors.d.ts +60 -0
- package/dist-types/src/commonErrors.d.ts.map +1 -0
- package/dist-types/src/configContext.d.ts +44 -0
- package/dist-types/src/configContext.d.ts.map +1 -0
- package/dist-types/src/downloadEntityFromRepo.d.ts +11 -0
- package/dist-types/src/downloadEntityFromRepo.d.ts.map +1 -0
- package/dist-types/src/errors.d.ts +10 -0
- package/dist-types/src/errors.d.ts.map +1 -0
- package/dist-types/src/getPathContents/ParsedMetaInfoAboutPathContentsFromGitHubAPI.d.ts +141 -0
- package/dist-types/src/getPathContents/ParsedMetaInfoAboutPathContentsFromGitHubAPI.d.ts.map +1 -0
- package/dist-types/src/getPathContents/PathContentsMetaInfo.d.ts +66 -0
- package/dist-types/src/getPathContents/PathContentsMetaInfo.d.ts.map +1 -0
- package/dist-types/src/getPathContents/RawStreamOfRepoPathContentsFromGitHubAPI.d.ts +9 -0
- package/dist-types/src/getPathContents/RawStreamOfRepoPathContentsFromGitHubAPI.d.ts.map +1 -0
- package/dist-types/src/getPathContents/RepoPathContentsFromGitHubAPI.d.ts +81 -0
- package/dist-types/src/getPathContents/RepoPathContentsFromGitHubAPI.d.ts.map +1 -0
- package/dist-types/src/getPathContents/index.d.ts +5 -0
- package/dist-types/src/getPathContents/index.d.ts.map +1 -0
- package/dist-types/src/getPathContents/parseGitLFSObjectEither.d.ts +40 -0
- package/dist-types/src/getPathContents/parseGitLFSObjectEither.d.ts.map +1 -0
- package/dist-types/src/getReadableTarGzStreamOfRepoDirectory.d.ts +13 -0
- package/dist-types/src/getReadableTarGzStreamOfRepoDirectory.d.ts.map +1 -0
- package/dist-types/src/index.d.ts +9 -0
- package/dist-types/src/index.d.ts.map +1 -0
- package/dist-types/src/octokit.d.ts +8 -0
- package/dist-types/src/octokit.d.ts.map +1 -0
- package/dist-types/src/unpackRepoFolderTarGzStreamToFs.d.ts +16 -0
- package/dist-types/src/unpackRepoFolderTarGzStreamToFs.d.ts.map +1 -0
- package/dist-types/src/writeFileStreamToDestinationPath.d.ts +16 -0
- package/dist-types/src/writeFileStreamToDestinationPath.d.ts.map +1 -0
- package/package.json +139 -0
- package/src/TaggedErrorVerifyingCause.ts +142 -0
- package/src/castToReadableStream.ts +44 -0
- package/src/cli.ts +14 -0
- package/src/commandLineParams.ts +257 -0
- package/src/commonErrors.ts +139 -0
- package/src/configContext.ts +46 -0
- package/src/downloadEntityFromRepo.ts +86 -0
- package/src/errors.ts +24 -0
- package/src/getPathContents/ParsedMetaInfoAboutPathContentsFromGitHubAPI.ts +76 -0
- package/src/getPathContents/PathContentsMetaInfo.ts +85 -0
- package/src/getPathContents/RawStreamOfRepoPathContentsFromGitHubAPI.ts +6 -0
- package/src/getPathContents/RepoPathContentsFromGitHubAPI.ts +82 -0
- package/src/getPathContents/index.ts +7 -0
- package/src/getPathContents/parseGitLFSObjectEither.ts +143 -0
- package/src/getReadableTarGzStreamOfRepoDirectory.ts +65 -0
- package/src/index.ts +13 -0
- package/src/octokit.ts +15 -0
- package/src/unpackRepoFolderTarGzStreamToFs.ts +61 -0
- package/src/writeFileStreamToDestinationPath.ts +45 -0
- package/template.env +13 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { outdent } from 'outdent'
|
|
2
|
+
|
|
3
|
+
import * as CLIOptions from '@effect/cli/Options'
|
|
4
|
+
import * as Path from '@effect/platform/Path'
|
|
5
|
+
import * as Config from 'effect/Config'
|
|
6
|
+
import * as Effect from 'effect/Effect'
|
|
7
|
+
import * as EFunction from 'effect/Function'
|
|
8
|
+
import * as ParseResult from 'effect/ParseResult'
|
|
9
|
+
import * as Schema from 'effect/Schema'
|
|
10
|
+
|
|
11
|
+
const isGitHubSlug = (s: string) => !!s.match(/^[a-z0-9.\-_]+$/gi)
|
|
12
|
+
|
|
13
|
+
const invalidGitHubSlugMessage =
|
|
14
|
+
'GitHub handle should have only ASCII letters, digits, and the characters ".", "-", and "_"'
|
|
15
|
+
|
|
16
|
+
// https://developer.mozilla.org/en-US/docs/Glossary/Slug
|
|
17
|
+
const GitHubSlugStringSchema = Schema.NonEmptyString.pipe(
|
|
18
|
+
Schema.filter(s => isGitHubSlug(s) || invalidGitHubSlugMessage),
|
|
19
|
+
// TODO brandify this
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
// TODO change approach to default values. Either remove defaults completely or
|
|
23
|
+
// provided an easy way to set for people their own defaults instead of
|
|
24
|
+
// comparing them to the hardcoded default value. Also document the helpers for
|
|
25
|
+
// overriding defaults in TSDoc of exported CLIOptions objects
|
|
26
|
+
|
|
27
|
+
const withGitHubSlugConfigValidation = Config.validate({
|
|
28
|
+
message: invalidGitHubSlugMessage,
|
|
29
|
+
validation: isGitHubSlug,
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
const pathToEntityInRepoDescription = 'Path to file or directory in repo'
|
|
33
|
+
|
|
34
|
+
const repoOwnerDescription = outdent`
|
|
35
|
+
This is a username (login handle) of a person owning repo you
|
|
36
|
+
are trying to download from. For example, if the repository's URL is
|
|
37
|
+
\`https://github.com/apache/superset\`, the owner is \`apache\`
|
|
38
|
+
`
|
|
39
|
+
|
|
40
|
+
const repoNameDescription = outdent`
|
|
41
|
+
This is the name handle of the repository you are trying to download
|
|
42
|
+
from. For example, if the repository's URL is
|
|
43
|
+
\`https://github.com/apache/superset\`, the name is \`superset\`
|
|
44
|
+
`
|
|
45
|
+
|
|
46
|
+
const destinationPathDescription = outdent`
|
|
47
|
+
Local path of the downloaded file or directory. If
|
|
48
|
+
"pathToEntityInRepo" points to a file, then last element of the
|
|
49
|
+
destination path will be new file name. If "pathToEntityInRepo" points
|
|
50
|
+
to a directory then all files and directories inside directory at
|
|
51
|
+
"pathToEntityInRepo" will be put into a directory with name equal last
|
|
52
|
+
element of destination path. If the directory doesn't exist, it will
|
|
53
|
+
be automatically created.
|
|
54
|
+
`
|
|
55
|
+
|
|
56
|
+
const gitRefDescription = outdent`
|
|
57
|
+
This is the commit's SHA hash, branch name, tag name, or any other ref
|
|
58
|
+
you want to download from. If you don't specify it, the default branch
|
|
59
|
+
in the repository will be used.
|
|
60
|
+
`
|
|
61
|
+
|
|
62
|
+
const RepoNameConfig = EFunction.pipe(
|
|
63
|
+
Config.nonEmptyString('REPO_NAME'),
|
|
64
|
+
withGitHubSlugConfigValidation,
|
|
65
|
+
Config.withDescription(repoNameDescription),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
const RepoOwnerConfig = EFunction.pipe(
|
|
69
|
+
Config.nonEmptyString('REPO_OWNER'),
|
|
70
|
+
withGitHubSlugConfigValidation,
|
|
71
|
+
Config.withDescription(repoOwnerDescription),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
const DestinationPathConfig = EFunction.pipe(
|
|
75
|
+
Config.nonEmptyString('DESTINATION_PATH'),
|
|
76
|
+
Config.withDefault('./destination'),
|
|
77
|
+
Config.withDescription(destinationPathDescription),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
const PathToEntityInRepoConfig = EFunction.pipe(
|
|
81
|
+
Config.nonEmptyString('PATH_TO_ENTITY_IN_REPO'),
|
|
82
|
+
Config.withDefault('.'),
|
|
83
|
+
Config.withDescription(pathToEntityInRepoDescription),
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
const GitRefConfig = EFunction.pipe(
|
|
87
|
+
Config.nonEmptyString('GIT_REF'),
|
|
88
|
+
Config.withDefault('HEAD'),
|
|
89
|
+
Config.withDescription(gitRefDescription),
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
const CleanRepoEntityPathString = Schema.transformOrFail(
|
|
93
|
+
Schema.NonEmptyString,
|
|
94
|
+
Schema.NonEmptyString,
|
|
95
|
+
{
|
|
96
|
+
strict: true,
|
|
97
|
+
decode: (dirtyPathToEntityInRepo, _, ast) =>
|
|
98
|
+
Effect.flatMap(Path.Path, path => {
|
|
99
|
+
// dot can be there only when that's all there is. path.join(...)
|
|
100
|
+
// removes all './', so '.' will never be just left by themself. If it's
|
|
101
|
+
// there, it's very intentional and no other elements in the path exist.
|
|
102
|
+
const cleanPathToEntityInRepo = path
|
|
103
|
+
.join(dirtyPathToEntityInRepo)
|
|
104
|
+
.replaceAll(/\/?$/g, '')
|
|
105
|
+
|
|
106
|
+
if (cleanPathToEntityInRepo.startsWith('..'))
|
|
107
|
+
return ParseResult.fail(
|
|
108
|
+
new ParseResult.Type(
|
|
109
|
+
ast,
|
|
110
|
+
dirtyPathToEntityInRepo,
|
|
111
|
+
"Can't request contents that lie higher than the root of the repo",
|
|
112
|
+
),
|
|
113
|
+
)
|
|
114
|
+
return ParseResult.succeed(cleanPathToEntityInRepo)
|
|
115
|
+
}),
|
|
116
|
+
encode: ParseResult.succeed,
|
|
117
|
+
},
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Text parameter containing path to a directory or a file inside target repo.
|
|
122
|
+
*
|
|
123
|
+
* Can be passed in two ways:
|
|
124
|
+
* 1. As CLI option `gdl --pathToEntityInRepo nestedFolder/Readme.md`
|
|
125
|
+
* 2. As env variable `PATH_TO_ENTITY_IN_REPO="nestedFolder/Readme.md" gdl`
|
|
126
|
+
*
|
|
127
|
+
* Has default: `.`, which means that if not specified, script will download
|
|
128
|
+
* entire repository (download root directory of the repository)
|
|
129
|
+
*
|
|
130
|
+
* Parameter is automatically validated to not point higher than the root of the
|
|
131
|
+
* repository.
|
|
132
|
+
*
|
|
133
|
+
* @since 0.1.7
|
|
134
|
+
* @category CLI options
|
|
135
|
+
* @constant
|
|
136
|
+
*/
|
|
137
|
+
export const pathToEntityInRepoCLIOptionBackedByEnv: CLIOptions.Options<string> =
|
|
138
|
+
EFunction.pipe(
|
|
139
|
+
CLIOptions.text(`pathToEntityInRepo`),
|
|
140
|
+
CLIOptions.withDescription(pathToEntityInRepoDescription),
|
|
141
|
+
CLIOptions.withFallbackConfig(PathToEntityInRepoConfig),
|
|
142
|
+
CLIOptions.withSchema(CleanRepoEntityPathString),
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Text parameter containing URL slug of the user which owns the repo.
|
|
147
|
+
*
|
|
148
|
+
* Examples:
|
|
149
|
+
* 1. `apache`
|
|
150
|
+
* 2. `nikelborm`
|
|
151
|
+
*
|
|
152
|
+
* Can be passed in two ways:
|
|
153
|
+
* 1. As CLI option `gdl --repoOwner apache`
|
|
154
|
+
* 2. As env variable `REPO_OWNER="apache" gdl`
|
|
155
|
+
*
|
|
156
|
+
* Doesn`t have defaults and will fail if not specified.
|
|
157
|
+
*
|
|
158
|
+
* Parameter is automatically validated so it can consist of only ASCII letters,
|
|
159
|
+
* digits, and the characters `.`, `-`, and `_`.
|
|
160
|
+
*
|
|
161
|
+
* @since 0.1.7
|
|
162
|
+
* @category CLI options
|
|
163
|
+
* @constant
|
|
164
|
+
*/
|
|
165
|
+
export const repoOwnerCLIOptionBackedByEnv: CLIOptions.Options<string> =
|
|
166
|
+
EFunction.pipe(
|
|
167
|
+
CLIOptions.text(`repoOwner`),
|
|
168
|
+
CLIOptions.withDescription(repoOwnerDescription),
|
|
169
|
+
CLIOptions.withFallbackConfig(RepoOwnerConfig),
|
|
170
|
+
CLIOptions.withSchema(GitHubSlugStringSchema),
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Text parameter containing URL slug of the repo itself.
|
|
175
|
+
*
|
|
176
|
+
* Examples:
|
|
177
|
+
* 1. `superset`
|
|
178
|
+
* 2. `git-dl`
|
|
179
|
+
*
|
|
180
|
+
* Can be passed in two ways:
|
|
181
|
+
* 1. As CLI option `gdl --repoName superset`
|
|
182
|
+
* 2. As env variable `REPO_NAME="superset" gdl`
|
|
183
|
+
*
|
|
184
|
+
* Doesn`t have defaults and will fail if not specified.
|
|
185
|
+
*
|
|
186
|
+
* Parameter is automatically validated so it can consist of only ASCII letters,
|
|
187
|
+
* digits, and the characters `.`, `-`, and `_`.
|
|
188
|
+
*
|
|
189
|
+
* @since 0.1.7
|
|
190
|
+
* @category CLI options
|
|
191
|
+
* @constant
|
|
192
|
+
*/
|
|
193
|
+
export const repoNameCLIOptionBackedByEnv: CLIOptions.Options<string> =
|
|
194
|
+
EFunction.pipe(
|
|
195
|
+
CLIOptions.text(`repoName`),
|
|
196
|
+
CLIOptions.withDescription(repoNameDescription),
|
|
197
|
+
CLIOptions.withFallbackConfig(RepoNameConfig),
|
|
198
|
+
CLIOptions.withSchema(GitHubSlugStringSchema),
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Text parameter containing path inside your local file system, your new
|
|
203
|
+
* file/directory will be placed at. Last element of the path will be the name
|
|
204
|
+
* of the new file/directory.
|
|
205
|
+
*
|
|
206
|
+
* Examples:
|
|
207
|
+
* 1. `../docker`
|
|
208
|
+
* 2. `/tmp/Readme.md`
|
|
209
|
+
*
|
|
210
|
+
* Can be passed in two ways:
|
|
211
|
+
* 1. As CLI option `gdl --destinationPath docker`
|
|
212
|
+
* 2. As env variable `DESTINATION_PATH="docker" gdl`
|
|
213
|
+
*
|
|
214
|
+
* Has default: `./destination`, which means that if not specified, script will
|
|
215
|
+
* either create a file or a directory named `destination` inside your current PWD
|
|
216
|
+
* depending on the type of remote target.
|
|
217
|
+
*
|
|
218
|
+
* @since 0.1.7
|
|
219
|
+
* @category CLI options
|
|
220
|
+
* @constant
|
|
221
|
+
* @readonly
|
|
222
|
+
*/
|
|
223
|
+
export const destinationPathCLIOptionBackedByEnv: CLIOptions.Options<string> =
|
|
224
|
+
EFunction.pipe(
|
|
225
|
+
CLIOptions.text(`destinationPath`),
|
|
226
|
+
CLIOptions.withDescription(destinationPathDescription),
|
|
227
|
+
CLIOptions.withFallbackConfig(DestinationPathConfig),
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Text parameter containing commit SHA hash, branch name, or tag name you want
|
|
232
|
+
* to download from.
|
|
233
|
+
*
|
|
234
|
+
* Examples:
|
|
235
|
+
* 1. `HEAD`
|
|
236
|
+
* 2. `main`
|
|
237
|
+
* 3. `4.1.1`
|
|
238
|
+
* 4. `dca3efb3dd2a2a75aea32e3561c4104a53f02808`
|
|
239
|
+
* 5. `dca3efb`
|
|
240
|
+
*
|
|
241
|
+
* Can be passed in two ways:
|
|
242
|
+
* 1. As CLI option `gdl --gitRef 4.1.1`
|
|
243
|
+
* 2. As env variable `GIT_REF="4.1.1" gdl`
|
|
244
|
+
*
|
|
245
|
+
* Has default: `HEAD`, which means that if not specified, the default branch in
|
|
246
|
+
* the repository will be used.
|
|
247
|
+
*
|
|
248
|
+
* @since 0.1.7
|
|
249
|
+
* @category CLI options
|
|
250
|
+
* @constant
|
|
251
|
+
*/
|
|
252
|
+
export const gitRefCLIOptionBackedByEnv: CLIOptions.Options<string> =
|
|
253
|
+
EFunction.pipe(
|
|
254
|
+
CLIOptions.text(`gitRef`),
|
|
255
|
+
CLIOptions.withDescription(gitRefDescription),
|
|
256
|
+
CLIOptions.withFallbackConfig(GitRefConfig),
|
|
257
|
+
)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { RequestError } from '@octokit/request-error'
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
buildTaggedErrorClassVerifyingCause,
|
|
5
|
+
type TaggedErrorClass,
|
|
6
|
+
} from './TaggedErrorVerifyingCause.ts'
|
|
7
|
+
|
|
8
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
9
|
+
// library will have much faster type inference
|
|
10
|
+
const _1: TaggedErrorClass<{
|
|
11
|
+
ErrorName: 'GitHubApiGeneralServerError'
|
|
12
|
+
ExpectedCauseClass: typeof RequestError
|
|
13
|
+
}> = buildTaggedErrorClassVerifyingCause()(
|
|
14
|
+
'GitHubApiGeneralServerError',
|
|
15
|
+
'GitHub API Error: Bad server',
|
|
16
|
+
RequestError,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
export class GitHubApiGeneralServerError extends _1 {}
|
|
20
|
+
|
|
21
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
22
|
+
// library will have much faster type inference
|
|
23
|
+
const _2: TaggedErrorClass<{
|
|
24
|
+
ErrorName: 'GitHubApiGeneralUserError'
|
|
25
|
+
ExpectedCauseClass: typeof RequestError
|
|
26
|
+
DynamicContext: { readonly notes?: string }
|
|
27
|
+
}> = buildTaggedErrorClassVerifyingCause<{ readonly notes?: string }>()(
|
|
28
|
+
'GitHubApiGeneralUserError',
|
|
29
|
+
'GitHub API Error: Bad user, invalid request',
|
|
30
|
+
RequestError,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
export class GitHubApiGeneralUserError extends _2 {}
|
|
34
|
+
|
|
35
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
36
|
+
// library will have much faster type inference
|
|
37
|
+
const _3: TaggedErrorClass<{
|
|
38
|
+
ErrorName: 'GitHubApiThingNotExistsOrYouDontHaveAccessError'
|
|
39
|
+
ExpectedCauseClass: typeof RequestError
|
|
40
|
+
}> = buildTaggedErrorClassVerifyingCause()(
|
|
41
|
+
'GitHubApiThingNotExistsOrYouDontHaveAccessError',
|
|
42
|
+
"GitHub API Error: Either repo, owner, path in repo, or specified ref don't exist or you don't have permissions to access it",
|
|
43
|
+
RequestError,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
export class GitHubApiThingNotExistsOrYouDontHaveAccessError extends _3 {}
|
|
47
|
+
|
|
48
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
49
|
+
// library will have much faster type inference
|
|
50
|
+
const _4: TaggedErrorClass<{
|
|
51
|
+
ErrorName: 'GitHubApiRepoIsEmptyError'
|
|
52
|
+
ExpectedCauseClass: typeof RequestError
|
|
53
|
+
}> = buildTaggedErrorClassVerifyingCause()(
|
|
54
|
+
'GitHubApiRepoIsEmptyError',
|
|
55
|
+
'GitHub API Error: This Repo is empty',
|
|
56
|
+
RequestError,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
export class GitHubApiRepoIsEmptyError extends _4 {}
|
|
60
|
+
|
|
61
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
62
|
+
// library will have much faster type inference
|
|
63
|
+
const _5: TaggedErrorClass<{
|
|
64
|
+
ErrorName: 'GitHubApiNoCommitFoundForGitRefError'
|
|
65
|
+
ExpectedCauseClass: typeof RequestError
|
|
66
|
+
DynamicContext: { gitRef: string }
|
|
67
|
+
}> = buildTaggedErrorClassVerifyingCause<{ gitRef: string }>()(
|
|
68
|
+
'GitHubApiNoCommitFoundForGitRefError',
|
|
69
|
+
'GitHub API Error: No commit found for this git ref',
|
|
70
|
+
RequestError,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
export class GitHubApiNoCommitFoundForGitRefError extends _5 {}
|
|
74
|
+
|
|
75
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
76
|
+
// library will have much faster type inference
|
|
77
|
+
const _6: TaggedErrorClass<{
|
|
78
|
+
ErrorName: 'GitHubApiBadCredentialsError'
|
|
79
|
+
ExpectedCauseClass: typeof RequestError
|
|
80
|
+
}> = buildTaggedErrorClassVerifyingCause()(
|
|
81
|
+
'GitHubApiBadCredentialsError',
|
|
82
|
+
"GitHub API Error: Token you're using is invalid.",
|
|
83
|
+
RequestError,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
export class GitHubApiBadCredentialsError extends _6 {}
|
|
87
|
+
|
|
88
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
89
|
+
// library will have much faster type inference
|
|
90
|
+
const _7: TaggedErrorClass<{
|
|
91
|
+
ErrorName: 'GitHubApiAuthRatelimitedError'
|
|
92
|
+
ExpectedCauseClass: typeof RequestError
|
|
93
|
+
}> = buildTaggedErrorClassVerifyingCause()(
|
|
94
|
+
'GitHubApiAuthRatelimitedError',
|
|
95
|
+
'GitHub API Error: Too many invalid auth attempts. Chillout pal',
|
|
96
|
+
RequestError,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
export class GitHubApiAuthRatelimitedError extends _7 {}
|
|
100
|
+
|
|
101
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
102
|
+
// library will have much faster type inference
|
|
103
|
+
const _8: TaggedErrorClass<{
|
|
104
|
+
ErrorName: 'GitHubApiRatelimitedError'
|
|
105
|
+
ExpectedCauseClass: typeof RequestError
|
|
106
|
+
}> = buildTaggedErrorClassVerifyingCause()(
|
|
107
|
+
'GitHubApiRatelimitedError',
|
|
108
|
+
'GitHub API Error: Too many requests. Chillout pal',
|
|
109
|
+
RequestError,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
export class GitHubApiRatelimitedError extends _8 {}
|
|
113
|
+
|
|
114
|
+
export const parseCommonGitHubApiErrors = (error: RequestError) => {
|
|
115
|
+
if (error.status === 401) return new GitHubApiBadCredentialsError(error)
|
|
116
|
+
|
|
117
|
+
// https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#failed-login-limit
|
|
118
|
+
if (error.status === 403) return new GitHubApiAuthRatelimitedError(error)
|
|
119
|
+
|
|
120
|
+
// docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28
|
|
121
|
+
if (error.status === 429) return new GitHubApiRatelimitedError(error)
|
|
122
|
+
|
|
123
|
+
if (error.status === 404)
|
|
124
|
+
return new GitHubApiThingNotExistsOrYouDontHaveAccessError(error)
|
|
125
|
+
|
|
126
|
+
if (error.status >= 500) return new GitHubApiGeneralServerError(error)
|
|
127
|
+
|
|
128
|
+
if (error.status >= 400) return new GitHubApiGeneralUserError(error, {})
|
|
129
|
+
|
|
130
|
+
return error
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export type GitHubApiCommonErrors =
|
|
134
|
+
| RequestError
|
|
135
|
+
| GitHubApiGeneralServerError
|
|
136
|
+
| GitHubApiGeneralUserError
|
|
137
|
+
| GitHubApiBadCredentialsError
|
|
138
|
+
| GitHubApiAuthRatelimitedError
|
|
139
|
+
| GitHubApiRatelimitedError
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import * as Context from 'effect/Context'
|
|
2
|
+
import * as Effect from 'effect/Effect'
|
|
3
|
+
import * as Layer from 'effect/Layer'
|
|
4
|
+
|
|
5
|
+
export type InputConfig = Readonly<{
|
|
6
|
+
repo: Readonly<{
|
|
7
|
+
owner: string
|
|
8
|
+
name: string
|
|
9
|
+
}>
|
|
10
|
+
pathToEntityInRepo: string
|
|
11
|
+
gitRef: string
|
|
12
|
+
}>
|
|
13
|
+
|
|
14
|
+
export const InputConfigTag = Context.GenericTag<InputConfig>('InputConfig')
|
|
15
|
+
|
|
16
|
+
export type OutputConfig = Readonly<{
|
|
17
|
+
localPathAtWhichEntityFromRepoWillBeAvailable: string
|
|
18
|
+
}>
|
|
19
|
+
|
|
20
|
+
export const OutputConfigTag = Context.GenericTag<OutputConfig>('OutputConfig')
|
|
21
|
+
|
|
22
|
+
const InputConfigLive = (inputConfig: InputConfig) =>
|
|
23
|
+
Layer.succeed(InputConfigTag, InputConfigTag.of(inputConfig))
|
|
24
|
+
|
|
25
|
+
export const provideInputConfig = (inputConfig: InputConfig) =>
|
|
26
|
+
Effect.provide(InputConfigLive(inputConfig))
|
|
27
|
+
|
|
28
|
+
const OutputConfigLive = (outputConfig: OutputConfig) =>
|
|
29
|
+
Layer.succeed(OutputConfigTag, OutputConfigTag.of(outputConfig))
|
|
30
|
+
|
|
31
|
+
export type SingleTargetConfig = InputConfig & OutputConfig
|
|
32
|
+
|
|
33
|
+
export const provideSingleDownloadTargetConfig = ({
|
|
34
|
+
localPathAtWhichEntityFromRepoWillBeAvailable,
|
|
35
|
+
...inputConfig
|
|
36
|
+
}: SingleTargetConfig): (<A, E, R>(
|
|
37
|
+
self: Effect.Effect<A, E, R>,
|
|
38
|
+
) => Effect.Effect<A, E, Exclude<R, InputConfig | OutputConfig>>) =>
|
|
39
|
+
Effect.provide(
|
|
40
|
+
Layer.merge(
|
|
41
|
+
InputConfigLive(inputConfig),
|
|
42
|
+
OutputConfigLive({
|
|
43
|
+
localPathAtWhichEntityFromRepoWillBeAvailable,
|
|
44
|
+
}),
|
|
45
|
+
),
|
|
46
|
+
)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { Octokit } from '@octokit/core'
|
|
2
|
+
|
|
3
|
+
import type * as Cause from 'effect/Cause'
|
|
4
|
+
import * as Effect from 'effect/Effect'
|
|
5
|
+
|
|
6
|
+
import type { FailedToCastDataToReadableStreamError } from './castToReadableStream.ts'
|
|
7
|
+
import type {
|
|
8
|
+
GitHubApiAuthRatelimitedError,
|
|
9
|
+
GitHubApiBadCredentialsError,
|
|
10
|
+
GitHubApiGeneralServerError,
|
|
11
|
+
GitHubApiGeneralUserError,
|
|
12
|
+
GitHubApiNoCommitFoundForGitRefError,
|
|
13
|
+
GitHubApiRatelimitedError,
|
|
14
|
+
GitHubApiRepoIsEmptyError,
|
|
15
|
+
GitHubApiThingNotExistsOrYouDontHaveAccessError,
|
|
16
|
+
} from './commonErrors.ts'
|
|
17
|
+
import {
|
|
18
|
+
provideSingleDownloadTargetConfig,
|
|
19
|
+
type SingleTargetConfig,
|
|
20
|
+
} from './configContext.ts'
|
|
21
|
+
import {
|
|
22
|
+
type FailedToParseResponseFromRepoPathContentsMetaInfoAPIError,
|
|
23
|
+
type InconsistentExpectedAndRealContentSizeError,
|
|
24
|
+
PathContentsMetaInfo,
|
|
25
|
+
RawStreamOfRepoPathContentsFromGitHubAPI,
|
|
26
|
+
} from './getPathContents/index.ts'
|
|
27
|
+
import { getReadableTarGzStreamOfRepoDirectory } from './getReadableTarGzStreamOfRepoDirectory.ts'
|
|
28
|
+
import {
|
|
29
|
+
type FailedToUnpackRepoFolderTarGzStreamToFsError,
|
|
30
|
+
unpackRepoFolderTarGzStreamToFs,
|
|
31
|
+
} from './unpackRepoFolderTarGzStreamToFs.ts'
|
|
32
|
+
import {
|
|
33
|
+
type FailedToWriteFileStreamToDestinationPathError,
|
|
34
|
+
writeFileStreamToDestinationPath,
|
|
35
|
+
} from './writeFileStreamToDestinationPath.ts'
|
|
36
|
+
|
|
37
|
+
const downloadEntityFromRepoWithoutContext = Effect.gen(function* () {
|
|
38
|
+
const pathContentsMetaInfo = yield* PathContentsMetaInfo
|
|
39
|
+
|
|
40
|
+
if (pathContentsMetaInfo.type === 'dir')
|
|
41
|
+
return yield* unpackRepoFolderTarGzStreamToFs(
|
|
42
|
+
getReadableTarGzStreamOfRepoDirectory(pathContentsMetaInfo.treeSha),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
if (
|
|
46
|
+
pathContentsMetaInfo.meta ===
|
|
47
|
+
'This file is small enough that GitHub API decided to inline it'
|
|
48
|
+
)
|
|
49
|
+
return yield* writeFileStreamToDestinationPath(
|
|
50
|
+
pathContentsMetaInfo.contentStream,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
if (pathContentsMetaInfo.meta === 'This file can be downloaded as a blob')
|
|
54
|
+
return yield* writeFileStreamToDestinationPath(
|
|
55
|
+
RawStreamOfRepoPathContentsFromGitHubAPI,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
yield* Effect.dieMessage('LFS files are not yet supported')
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
62
|
+
// library will have much faster type inference
|
|
63
|
+
export const downloadEntityFromRepo = (
|
|
64
|
+
target: SingleTargetConfig,
|
|
65
|
+
): Effect.Effect<
|
|
66
|
+
void,
|
|
67
|
+
| Error
|
|
68
|
+
| InconsistentExpectedAndRealContentSizeError
|
|
69
|
+
| FailedToWriteFileStreamToDestinationPathError
|
|
70
|
+
| FailedToUnpackRepoFolderTarGzStreamToFsError
|
|
71
|
+
| Cause.UnknownException
|
|
72
|
+
| GitHubApiRepoIsEmptyError
|
|
73
|
+
| GitHubApiNoCommitFoundForGitRefError
|
|
74
|
+
| GitHubApiThingNotExistsOrYouDontHaveAccessError
|
|
75
|
+
| GitHubApiBadCredentialsError
|
|
76
|
+
| GitHubApiAuthRatelimitedError
|
|
77
|
+
| GitHubApiRatelimitedError
|
|
78
|
+
| GitHubApiGeneralServerError
|
|
79
|
+
| GitHubApiGeneralUserError
|
|
80
|
+
| FailedToParseResponseFromRepoPathContentsMetaInfoAPIError
|
|
81
|
+
| FailedToCastDataToReadableStreamError,
|
|
82
|
+
Octokit
|
|
83
|
+
> =>
|
|
84
|
+
downloadEntityFromRepoWithoutContext.pipe(
|
|
85
|
+
provideSingleDownloadTargetConfig(target),
|
|
86
|
+
)
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
export { FailedToCastDataToReadableStreamError } from './castToReadableStream.ts'
|
|
6
|
+
export {
|
|
7
|
+
GitHubApiAuthRatelimitedError,
|
|
8
|
+
GitHubApiBadCredentialsError,
|
|
9
|
+
type GitHubApiCommonErrors,
|
|
10
|
+
GitHubApiGeneralServerError,
|
|
11
|
+
GitHubApiGeneralUserError,
|
|
12
|
+
GitHubApiNoCommitFoundForGitRefError,
|
|
13
|
+
GitHubApiRatelimitedError,
|
|
14
|
+
GitHubApiRepoIsEmptyError,
|
|
15
|
+
GitHubApiThingNotExistsOrYouDontHaveAccessError,
|
|
16
|
+
} from './commonErrors.ts'
|
|
17
|
+
export {
|
|
18
|
+
FailedToParseGitLFSInfoError,
|
|
19
|
+
FailedToParseResponseFromRepoPathContentsMetaInfoAPIError,
|
|
20
|
+
InconsistentExpectedAndRealContentSizeError,
|
|
21
|
+
} from './getPathContents/index.ts'
|
|
22
|
+
export * from './TaggedErrorVerifyingCause.ts'
|
|
23
|
+
export { FailedToUnpackRepoFolderTarGzStreamToFsError } from './unpackRepoFolderTarGzStreamToFs.ts'
|
|
24
|
+
export { FailedToWriteFileStreamToDestinationPathError } from './writeFileStreamToDestinationPath.ts'
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import * as Effect from 'effect/Effect'
|
|
2
|
+
import * as Either from 'effect/Either'
|
|
3
|
+
import * as ParseResult from 'effect/ParseResult'
|
|
4
|
+
import * as Schema from 'effect/Schema'
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
buildTaggedErrorClassVerifyingCause,
|
|
8
|
+
type TaggedErrorClass,
|
|
9
|
+
} from '../TaggedErrorVerifyingCause.ts'
|
|
10
|
+
import { RepoPathContentsFromGitHubAPI } from './RepoPathContentsFromGitHubAPI.ts'
|
|
11
|
+
|
|
12
|
+
export const UnparsedMetaInfoAboutPathContentsFromGitHubAPI =
|
|
13
|
+
RepoPathContentsFromGitHubAPI('object')
|
|
14
|
+
|
|
15
|
+
export const ParsedMetaInfoAboutPathContentsFromGitHubAPI = Effect.gen(
|
|
16
|
+
function* () {
|
|
17
|
+
const response = yield* UnparsedMetaInfoAboutPathContentsFromGitHubAPI
|
|
18
|
+
|
|
19
|
+
return yield* Either.mapLeft(
|
|
20
|
+
decodeResponse(response.data),
|
|
21
|
+
parseError =>
|
|
22
|
+
new FailedToParseResponseFromRepoPathContentsMetaInfoAPIError(
|
|
23
|
+
parseError,
|
|
24
|
+
{
|
|
25
|
+
response,
|
|
26
|
+
},
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
},
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
const GitSomethingFields = {
|
|
33
|
+
size: Schema.Number,
|
|
34
|
+
name: Schema.String,
|
|
35
|
+
path: Schema.String,
|
|
36
|
+
sha: Schema.String,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const dirLiteral = Schema.Literal('dir')
|
|
40
|
+
const nonDirLiterals = Schema.Literal('file', 'submodule', 'symlink')
|
|
41
|
+
|
|
42
|
+
export const ResponseSchema = Schema.Union(
|
|
43
|
+
Schema.Struct({
|
|
44
|
+
type: Schema.Literal('dir'),
|
|
45
|
+
entries: Schema.Struct({
|
|
46
|
+
type: Schema.Union(dirLiteral, nonDirLiterals),
|
|
47
|
+
...GitSomethingFields,
|
|
48
|
+
}).pipe(Schema.Array),
|
|
49
|
+
...GitSomethingFields,
|
|
50
|
+
}),
|
|
51
|
+
Schema.Struct({
|
|
52
|
+
type: Schema.Literal('file'),
|
|
53
|
+
encoding: Schema.Literal('base64', 'none'),
|
|
54
|
+
content: Schema.String,
|
|
55
|
+
...GitSomethingFields,
|
|
56
|
+
}),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
const decodeResponse = Schema.decodeUnknownEither(ResponseSchema, {
|
|
60
|
+
exact: true,
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
// Extracting to a separate type is required by JSR, so that consumers of the
|
|
64
|
+
// library will have much faster type inference
|
|
65
|
+
|
|
66
|
+
const _1: TaggedErrorClass<{
|
|
67
|
+
ErrorName: 'FailedToParseResponseFromRepoPathContentsMetaInfoAPI'
|
|
68
|
+
ExpectedCauseClass: typeof ParseResult.ParseError
|
|
69
|
+
DynamicContext: { response: unknown }
|
|
70
|
+
}> = buildTaggedErrorClassVerifyingCause<{ response: unknown }>()(
|
|
71
|
+
'FailedToParseResponseFromRepoPathContentsMetaInfoAPI',
|
|
72
|
+
`Failed to parse response from repo path contents meta info API`,
|
|
73
|
+
ParseResult.ParseError,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
export class FailedToParseResponseFromRepoPathContentsMetaInfoAPIError extends _1 {}
|