@infersec/conduit 1.99.2 → 1.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2450 -2153
- package/dist/cli.sea.cjs +2448 -2151
- package/dist/modelManagement/ModelManager.d.ts +1 -0
- package/dist/modelManagement/chatTemplate.d.ts +41 -0
- package/dist/requestHandlers/createConduitOpenAIAPIReferenceHandlers.d.ts +3 -1
- package/dist/utils/openai.d.ts +18 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
|
|
|
6
6
|
|
|
7
7
|
import require$$0$6, { EventEmitter as EventEmitter$1, once, addAbortListener, on, setMaxListeners } from 'node:events';
|
|
8
8
|
import require$$1$2, { spawn, ChildProcess, execFile, spawnSync } from 'node:child_process';
|
|
9
|
-
import path$1, { win32, posix,
|
|
9
|
+
import path$1, { join, win32, posix, dirname, resolve as resolve$1, basename, sep as sep$1 } from 'node:path';
|
|
10
10
|
import * as require$$3$4 from 'node:fs';
|
|
11
11
|
import require$$3__default, { existsSync, createWriteStream, statSync, readFileSync, appendFileSync, writeFileSync, createReadStream } from 'node:fs';
|
|
12
12
|
import process$3, { platform, hrtime, execPath, execArgv } from 'node:process';
|
|
@@ -45,12 +45,12 @@ import require$$2$4 from 'node:worker_threads';
|
|
|
45
45
|
import require$$0$k, { fileURLToPath } from 'node:url';
|
|
46
46
|
import require$$1$8 from 'node:async_hooks';
|
|
47
47
|
import require$$1$9 from 'node:console';
|
|
48
|
-
import require$$0$l, {
|
|
48
|
+
import require$$0$l, { readFile, rm, mkdir, writeFile, realpath, readlink, readdir, lstat, stat, unlink, rename, access, copyFile } from 'node:fs/promises';
|
|
49
49
|
import require$$2$5 from 'node:timers';
|
|
50
|
-
import { StringDecoder } from 'node:string_decoder';
|
|
51
|
-
import { pipeline, finished } from 'node:stream/promises';
|
|
52
50
|
import 'fs/promises';
|
|
53
51
|
import 'stream/promises';
|
|
52
|
+
import { StringDecoder } from 'node:string_decoder';
|
|
53
|
+
import { pipeline, finished } from 'node:stream/promises';
|
|
54
54
|
import os, { constants as constants$5, hostname as hostname$2 } from 'node:os';
|
|
55
55
|
import tty from 'node:tty';
|
|
56
56
|
import require$$0$m from 'child_process';
|
|
@@ -19986,7 +19986,39 @@ object$1({
|
|
|
19986
19986
|
supportsVision: boolean$1()
|
|
19987
19987
|
});
|
|
19988
19988
|
const LLMModelTaskTypeSchema = _enum$1(["text-generation", "embeddings"]);
|
|
19989
|
+
const ReasoningEffortSchema = _enum$1(["none", "minimal", "low", "medium", "high", "xhigh"]);
|
|
19990
|
+
const CHAT_TEMPLATE_DEFAULT_FILE_PATH = "chat_template.jinja";
|
|
19991
|
+
const CHAT_TEMPLATE_LOCAL_FILE_NAME = "infersec-chat-template.jinja";
|
|
19992
|
+
const ChatTemplateOverrideSchema = discriminatedUnion("type", [
|
|
19993
|
+
object$1({
|
|
19994
|
+
filePath: string$2().min(1).default(CHAT_TEMPLATE_DEFAULT_FILE_PATH),
|
|
19995
|
+
repo: string$2().min(1),
|
|
19996
|
+
type: literal("huggingface")
|
|
19997
|
+
}),
|
|
19998
|
+
object$1({
|
|
19999
|
+
content: string$2().min(1),
|
|
20000
|
+
type: literal("inline")
|
|
20001
|
+
})
|
|
20002
|
+
]);
|
|
20003
|
+
const ThinkingConfigSchema = object$1({
|
|
20004
|
+
enabled: boolean$1().optional(),
|
|
20005
|
+
effort: ReasoningEffortSchema.optional()
|
|
20006
|
+
});
|
|
20007
|
+
const ANTHROPIC_THINKING_BUDGET_EFFORT_TIERS = [
|
|
20008
|
+
{ effort: "low", maxBudgetTokens: 2048 },
|
|
20009
|
+
{ effort: "medium", maxBudgetTokens: 8192 },
|
|
20010
|
+
{ effort: "high", maxBudgetTokens: Number.POSITIVE_INFINITY }
|
|
20011
|
+
];
|
|
20012
|
+
function reasoningEffortForAnthropicBudget(budgetTokens) {
|
|
20013
|
+
for (const tier of ANTHROPIC_THINKING_BUDGET_EFFORT_TIERS) {
|
|
20014
|
+
if (budgetTokens <= tier.maxBudgetTokens) {
|
|
20015
|
+
return tier.effort;
|
|
20016
|
+
}
|
|
20017
|
+
}
|
|
20018
|
+
return "high";
|
|
20019
|
+
}
|
|
19989
20020
|
const LLMModelSchema = object$1({
|
|
20021
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable().optional(),
|
|
19990
20022
|
format: LLMModelFormatSchema,
|
|
19991
20023
|
id: string$2().min(1),
|
|
19992
20024
|
multimodalEnabled: boolean$1(),
|
|
@@ -20001,7 +20033,8 @@ const LLMModelSchema = object$1({
|
|
|
20001
20033
|
type: literal("huggingface")
|
|
20002
20034
|
})
|
|
20003
20035
|
]),
|
|
20004
|
-
taskType: LLMModelTaskTypeSchema
|
|
20036
|
+
taskType: LLMModelTaskTypeSchema,
|
|
20037
|
+
thinkingConfig: ThinkingConfigSchema.nullable().optional()
|
|
20005
20038
|
});
|
|
20006
20039
|
object$1({
|
|
20007
20040
|
filePath: string$2().min(1),
|
|
@@ -20120,7 +20153,8 @@ const ConduitStateSchema = z
|
|
|
20120
20153
|
])
|
|
20121
20154
|
.and(z.object({
|
|
20122
20155
|
activeRequestCount: z.number().int().nonnegative().optional(),
|
|
20123
|
-
timestamp: z.string().datetime()
|
|
20156
|
+
timestamp: z.string().datetime(),
|
|
20157
|
+
warnings: z.array(z.string()).optional()
|
|
20124
20158
|
}));
|
|
20125
20159
|
const ConduitState = z.preprocess(value => {
|
|
20126
20160
|
if (value === null) {
|
|
@@ -20659,6 +20693,10 @@ const ChatCompletionMessageSchema = object$1({
|
|
|
20659
20693
|
const ChatCompletionCreateParamsSchema = object$1({
|
|
20660
20694
|
messages: array(ChatCompletionMessageParamSchema),
|
|
20661
20695
|
model: string$2(),
|
|
20696
|
+
chat_template_kwargs: record(string$2(), unknown())
|
|
20697
|
+
.nullable()
|
|
20698
|
+
.optional()
|
|
20699
|
+
.describe("Additional chat template variables passed to the model's chat template at render time (e.g. reasoning_effort)"),
|
|
20662
20700
|
frequency_penalty: number$1().min(-2).max(2).nullable().optional(),
|
|
20663
20701
|
function_call: union([literal("none"), literal("auto"), object$1({ name: string$2() })])
|
|
20664
20702
|
.optional(),
|
|
@@ -20674,6 +20712,9 @@ const ChatCompletionCreateParamsSchema = object$1({
|
|
|
20674
20712
|
max_tokens: number$1().positive().nullable().optional(),
|
|
20675
20713
|
n: number$1().positive().nullable().optional(),
|
|
20676
20714
|
presence_penalty: number$1().min(-2).max(2).nullable().optional(),
|
|
20715
|
+
reasoning_effort: ReasoningEffortSchema.nullable()
|
|
20716
|
+
.optional()
|
|
20717
|
+
.describe("Reasoning effort hint for thinking models. Forwarded to the chat template when one is configured"),
|
|
20677
20718
|
response_format: object$1({
|
|
20678
20719
|
type: _enum$1(["text", "json_object"])
|
|
20679
20720
|
})
|
|
@@ -20955,6 +20996,7 @@ RoutingMethod.FirstAvailable;
|
|
|
20955
20996
|
});
|
|
20956
20997
|
|
|
20957
20998
|
object$1({
|
|
20999
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable().optional(),
|
|
20958
21000
|
format: LLMModelFormatSchema,
|
|
20959
21001
|
multimodalEnabled: boolean$1().optional(),
|
|
20960
21002
|
name: ResourceNameSchema,
|
|
@@ -20964,7 +21006,8 @@ object$1({
|
|
|
20964
21006
|
.refine(value => value.includes("/"), {
|
|
20965
21007
|
message: "Slug must be fully qualified (owner/repo)"
|
|
20966
21008
|
}),
|
|
20967
|
-
taskType: LLMModelTaskTypeSchema.optional()
|
|
21009
|
+
taskType: LLMModelTaskTypeSchema.optional(),
|
|
21010
|
+
thinkingConfig: ThinkingConfigSchema.nullable().optional()
|
|
20968
21011
|
});
|
|
20969
21012
|
object$1({
|
|
20970
21013
|
results: array(object$1({
|
|
@@ -20980,6 +21023,7 @@ object$1({
|
|
|
20980
21023
|
}))
|
|
20981
21024
|
});
|
|
20982
21025
|
object$1({
|
|
21026
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable(),
|
|
20983
21027
|
created: string$2(),
|
|
20984
21028
|
id: ULIDSchema,
|
|
20985
21029
|
modelFormat: LLMModelFormatSchema,
|
|
@@ -20997,12 +21041,15 @@ object$1({
|
|
|
20997
21041
|
updated: string$2()
|
|
20998
21042
|
})),
|
|
20999
21043
|
taskType: LLMModelTaskTypeSchema,
|
|
21044
|
+
thinkingConfig: ThinkingConfigSchema.nullable(),
|
|
21000
21045
|
updated: string$2()
|
|
21001
21046
|
});
|
|
21002
21047
|
object$1({
|
|
21048
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable().optional(),
|
|
21003
21049
|
multimodalEnabled: boolean$1().optional(),
|
|
21004
21050
|
name: ResourceNameSchema.optional(),
|
|
21005
|
-
taskType: LLMModelTaskTypeSchema.optional()
|
|
21051
|
+
taskType: LLMModelTaskTypeSchema.optional(),
|
|
21052
|
+
thinkingConfig: ThinkingConfigSchema.nullable().optional()
|
|
21006
21053
|
});
|
|
21007
21054
|
object$1({
|
|
21008
21055
|
success: literal(true)
|
|
@@ -112932,653 +112979,1858 @@ var createRouter = /*@__PURE__*/getDefaultExportFromCjs(expressPromiseRouterExpo
|
|
|
112932
112979
|
|
|
112933
112980
|
const SERVED_MODEL_NAME = "default";
|
|
112934
112981
|
|
|
112935
|
-
|
|
112936
|
-
"api-key",
|
|
112937
|
-
"auth-token",
|
|
112938
|
-
"hf-token",
|
|
112939
|
-
"key",
|
|
112940
|
-
"password",
|
|
112941
|
-
"secret",
|
|
112942
|
-
"token"
|
|
112943
|
-
]);
|
|
112944
|
-
function redactSecretArgs(args) {
|
|
112945
|
-
return args.map((arg, index) => {
|
|
112946
|
-
const equalsIndex = arg.indexOf("=");
|
|
112947
|
-
if (arg.startsWith("--") && equalsIndex > 0) {
|
|
112948
|
-
const flag = arg.slice(2, equalsIndex);
|
|
112949
|
-
if (SECRET_FLAGS.has(flag)) {
|
|
112950
|
-
return `${arg.slice(0, equalsIndex + 1)}***`;
|
|
112951
|
-
}
|
|
112952
|
-
}
|
|
112953
|
-
const previous = args[index - 1];
|
|
112954
|
-
if (previous &&
|
|
112955
|
-
previous.startsWith("--") &&
|
|
112956
|
-
!previous.includes("=") &&
|
|
112957
|
-
SECRET_FLAGS.has(previous.slice(2))) {
|
|
112958
|
-
return "***";
|
|
112959
|
-
}
|
|
112960
|
-
return arg;
|
|
112961
|
-
});
|
|
112962
|
-
}
|
|
112963
|
-
async function createEngineProcess({ args, bin, logger }) {
|
|
112964
|
-
logger.info("Starting engine process", {
|
|
112965
|
-
command: { args: redactSecretArgs(args), bin }
|
|
112966
|
-
});
|
|
112967
|
-
const processManager = new ProcessManager({ args, command: bin });
|
|
112968
|
-
await processManager.start();
|
|
112969
|
-
return processManager;
|
|
112970
|
-
}
|
|
112982
|
+
// src/lib/cache-management.ts
|
|
112971
112983
|
|
|
112972
|
-
|
|
112973
|
-
|
|
112974
|
-
|
|
112975
|
-
|
|
112984
|
+
// src/consts.ts
|
|
112985
|
+
var HUB_URL = "https://huggingface.co";
|
|
112986
|
+
|
|
112987
|
+
// src/error.ts
|
|
112988
|
+
async function createApiError(response, opts) {
|
|
112989
|
+
const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
|
|
112990
|
+
error.message = `Api error with status ${error.statusCode}${""}`;
|
|
112991
|
+
const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". ");
|
|
112992
|
+
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
|
|
112993
|
+
const json = await response.json();
|
|
112994
|
+
error.message = json.error || json.message || error.message;
|
|
112995
|
+
if (json.error_description) {
|
|
112996
|
+
error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description;
|
|
112976
112997
|
}
|
|
112977
|
-
|
|
112998
|
+
error.data = json;
|
|
112999
|
+
} else {
|
|
113000
|
+
error.data = { message: await response.text() };
|
|
113001
|
+
}
|
|
113002
|
+
error.message += `. ${trailer}`;
|
|
113003
|
+
throw error;
|
|
112978
113004
|
}
|
|
112979
|
-
|
|
112980
|
-
|
|
112981
|
-
|
|
112982
|
-
|
|
112983
|
-
|
|
112984
|
-
|
|
112985
|
-
|
|
112986
|
-
|
|
112987
|
-
|
|
112988
|
-
|
|
112989
|
-
|
|
112990
|
-
|
|
112991
|
-
|
|
112992
|
-
|
|
112993
|
-
|
|
112994
|
-
|
|
112995
|
-
|
|
112996
|
-
|
|
112997
|
-
|
|
112998
|
-
|
|
112999
|
-
|
|
113000
|
-
|
|
113001
|
-
|
|
113002
|
-
|
|
113003
|
-
|
|
113004
|
-
|
|
113005
|
-
|
|
113006
|
-
|
|
113007
|
-
|
|
113008
|
-
|
|
113009
|
-
|
|
113005
|
+
var HubApiError = class extends Error {
|
|
113006
|
+
statusCode;
|
|
113007
|
+
url;
|
|
113008
|
+
requestId;
|
|
113009
|
+
data;
|
|
113010
|
+
constructor(url, statusCode, requestId, message) {
|
|
113011
|
+
super(message);
|
|
113012
|
+
this.statusCode = statusCode;
|
|
113013
|
+
this.requestId = requestId;
|
|
113014
|
+
this.url = url;
|
|
113015
|
+
}
|
|
113016
|
+
};
|
|
113017
|
+
var InvalidApiResponseFormatError = class extends Error {
|
|
113018
|
+
};
|
|
113019
|
+
|
|
113020
|
+
// src/utils/checkCredentials.ts
|
|
113021
|
+
function checkAccessToken(accessToken) {
|
|
113022
|
+
if (!accessToken.startsWith("hf_")) {
|
|
113023
|
+
throw new TypeError("Your access token must start with 'hf_'");
|
|
113024
|
+
}
|
|
113025
|
+
}
|
|
113026
|
+
function checkCredentials(params) {
|
|
113027
|
+
if (params.accessToken) {
|
|
113028
|
+
checkAccessToken(params.accessToken);
|
|
113029
|
+
return params.accessToken;
|
|
113030
|
+
}
|
|
113031
|
+
if (params.credentials?.accessToken) {
|
|
113032
|
+
checkAccessToken(params.credentials.accessToken);
|
|
113033
|
+
return params.credentials.accessToken;
|
|
113034
|
+
}
|
|
113035
|
+
}
|
|
113036
|
+
|
|
113037
|
+
// src/utils/toRepoId.ts
|
|
113038
|
+
function toRepoId(repo) {
|
|
113039
|
+
if (typeof repo !== "string") {
|
|
113040
|
+
return repo;
|
|
113041
|
+
}
|
|
113042
|
+
if (repo.startsWith("model/") || repo.startsWith("models/")) {
|
|
113043
|
+
throw new TypeError(
|
|
113044
|
+
"A repo designation for a model should not start with 'models/', directly specify the model namespace / name"
|
|
113045
|
+
);
|
|
113046
|
+
}
|
|
113047
|
+
if (repo.startsWith("space/")) {
|
|
113048
|
+
throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'");
|
|
113049
|
+
}
|
|
113050
|
+
if (repo.startsWith("dataset/")) {
|
|
113051
|
+
throw new TypeError("Datasets should start with 'dataset/', plural, not 'dataset/'");
|
|
113052
|
+
}
|
|
113053
|
+
const slashes = repo.split("/").length - 1;
|
|
113054
|
+
if (repo.startsWith("spaces/")) {
|
|
113055
|
+
if (slashes !== 2) {
|
|
113056
|
+
throw new TypeError("Space Id must include namespace and name of the space");
|
|
113010
113057
|
}
|
|
113011
|
-
|
|
113012
|
-
|
|
113058
|
+
return {
|
|
113059
|
+
type: "space",
|
|
113060
|
+
name: repo.slice("spaces/".length)
|
|
113061
|
+
};
|
|
113062
|
+
}
|
|
113063
|
+
if (repo.startsWith("datasets/")) {
|
|
113064
|
+
if (slashes > 2) {
|
|
113065
|
+
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
113013
113066
|
}
|
|
113014
|
-
return
|
|
113067
|
+
return {
|
|
113068
|
+
type: "dataset",
|
|
113069
|
+
name: repo.slice("datasets/".length)
|
|
113070
|
+
};
|
|
113071
|
+
}
|
|
113072
|
+
if (slashes > 1) {
|
|
113073
|
+
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
113074
|
+
}
|
|
113075
|
+
return {
|
|
113076
|
+
type: "model",
|
|
113077
|
+
name: repo
|
|
113078
|
+
};
|
|
113015
113079
|
}
|
|
113080
|
+
new Promise((r) => {
|
|
113081
|
+
});
|
|
113016
113082
|
|
|
113017
|
-
|
|
113018
|
-
|
|
113019
|
-
|
|
113020
|
-
|
|
113021
|
-
|
|
113022
|
-
|
|
113023
|
-
|
|
113024
|
-
|
|
113025
|
-
|
|
113026
|
-
|
|
113027
|
-
|
|
113028
|
-
|
|
113029
|
-
|
|
113030
|
-
|
|
113031
|
-
|
|
113083
|
+
// src/utils/combineUint8Arrays.ts
|
|
113084
|
+
function combineUint8Arrays(a, b) {
|
|
113085
|
+
const aLength = a.length;
|
|
113086
|
+
const combinedBytes = new Uint8Array(aLength + b.length);
|
|
113087
|
+
combinedBytes.set(a);
|
|
113088
|
+
combinedBytes.set(b, aLength);
|
|
113089
|
+
return combinedBytes;
|
|
113090
|
+
}
|
|
113091
|
+
function readU64(b, n) {
|
|
113092
|
+
let x = 0;
|
|
113093
|
+
x |= b[n++] << 0;
|
|
113094
|
+
x |= b[n++] << 8;
|
|
113095
|
+
x |= b[n++] << 16;
|
|
113096
|
+
x |= b[n++] << 24;
|
|
113097
|
+
x |= b[n++] << 32;
|
|
113098
|
+
x |= b[n++] << 40;
|
|
113099
|
+
x |= b[n++] << 48;
|
|
113100
|
+
x |= b[n++] << 56;
|
|
113101
|
+
return x;
|
|
113102
|
+
}
|
|
113103
|
+
function readU32(b, n) {
|
|
113104
|
+
let x = 0;
|
|
113105
|
+
x |= b[n++] << 0;
|
|
113106
|
+
x |= b[n++] << 8;
|
|
113107
|
+
x |= b[n++] << 16;
|
|
113108
|
+
x |= b[n++] << 24;
|
|
113109
|
+
return x;
|
|
113110
|
+
}
|
|
113111
|
+
|
|
113112
|
+
// src/vendor/lz4js/index.ts
|
|
113113
|
+
var minMatch = 4;
|
|
113114
|
+
var hashSize = 1 << 16;
|
|
113115
|
+
makeHashTable();
|
|
113116
|
+
var magicNum = 407708164;
|
|
113117
|
+
var fdContentChksum = 4;
|
|
113118
|
+
var fdContentSize = 8;
|
|
113119
|
+
var fdBlockChksum = 16;
|
|
113120
|
+
var fdVersion = 64;
|
|
113121
|
+
var fdVersionMask = 192;
|
|
113122
|
+
var bsUncompressed = 2147483648;
|
|
113123
|
+
var bsShift = 4;
|
|
113124
|
+
var bsMask = 7;
|
|
113125
|
+
var bsMap = {
|
|
113126
|
+
4: 65536,
|
|
113127
|
+
5: 262144,
|
|
113128
|
+
6: 1048576,
|
|
113129
|
+
7: 4194304
|
|
113032
113130
|
};
|
|
113033
|
-
|
|
113034
|
-
|
|
113035
|
-
|
|
113036
|
-
|
|
113037
|
-
|
|
113038
|
-
|
|
113039
|
-
|
|
113040
|
-
return [ai, bi];
|
|
113041
|
-
}
|
|
113042
|
-
begs = [];
|
|
113043
|
-
left = str.length;
|
|
113044
|
-
while (i >= 0 && !result) {
|
|
113045
|
-
if (i === ai) {
|
|
113046
|
-
begs.push(i);
|
|
113047
|
-
ai = str.indexOf(a, i + 1);
|
|
113048
|
-
}
|
|
113049
|
-
else if (begs.length === 1) {
|
|
113050
|
-
const r = begs.pop();
|
|
113051
|
-
if (r !== undefined)
|
|
113052
|
-
result = [r, bi];
|
|
113053
|
-
}
|
|
113054
|
-
else {
|
|
113055
|
-
beg = begs.pop();
|
|
113056
|
-
if (beg !== undefined && beg < left) {
|
|
113057
|
-
left = beg;
|
|
113058
|
-
right = bi;
|
|
113059
|
-
}
|
|
113060
|
-
bi = str.indexOf(b, i + 1);
|
|
113061
|
-
}
|
|
113062
|
-
i = ai < bi && ai >= 0 ? ai : bi;
|
|
113063
|
-
}
|
|
113064
|
-
if (begs.length && right !== undefined) {
|
|
113065
|
-
result = [left, right];
|
|
113066
|
-
}
|
|
113131
|
+
function makeHashTable() {
|
|
113132
|
+
try {
|
|
113133
|
+
return new Uint32Array(hashSize);
|
|
113134
|
+
} catch (error) {
|
|
113135
|
+
const hashTable2 = new Array(hashSize);
|
|
113136
|
+
for (let i = 0; i < hashSize; i++) {
|
|
113137
|
+
hashTable2[i] = 0;
|
|
113067
113138
|
}
|
|
113068
|
-
return
|
|
113069
|
-
}
|
|
113070
|
-
|
|
113071
|
-
const escSlash = '\0SLASH' + Math.random() + '\0';
|
|
113072
|
-
const escOpen = '\0OPEN' + Math.random() + '\0';
|
|
113073
|
-
const escClose = '\0CLOSE' + Math.random() + '\0';
|
|
113074
|
-
const escComma = '\0COMMA' + Math.random() + '\0';
|
|
113075
|
-
const escPeriod = '\0PERIOD' + Math.random() + '\0';
|
|
113076
|
-
const escSlashPattern = new RegExp(escSlash, 'g');
|
|
113077
|
-
const escOpenPattern = new RegExp(escOpen, 'g');
|
|
113078
|
-
const escClosePattern = new RegExp(escClose, 'g');
|
|
113079
|
-
const escCommaPattern = new RegExp(escComma, 'g');
|
|
113080
|
-
const escPeriodPattern = new RegExp(escPeriod, 'g');
|
|
113081
|
-
const slashPattern = /\\\\/g;
|
|
113082
|
-
const openPattern = /\\{/g;
|
|
113083
|
-
const closePattern = /\\}/g;
|
|
113084
|
-
const commaPattern = /\\,/g;
|
|
113085
|
-
const periodPattern = /\\\./g;
|
|
113086
|
-
const EXPANSION_MAX = 100_000;
|
|
113087
|
-
// `EXPANSION_MAX` caps the *number* of expansions, but not their length. An
|
|
113088
|
-
// input like `'{a,b}'.repeat(1500)` stays under that count - its output is
|
|
113089
|
-
// truncated to 100k results - while making every result ~1500 characters
|
|
113090
|
-
// long. The result set, and the intermediate arrays built while combining
|
|
113091
|
-
// brace sets, then grow large enough to exhaust memory and crash the process
|
|
113092
|
-
// (CVE-2026-14257). `EXPANSION_MAX_LENGTH` bounds the total number of
|
|
113093
|
-
// characters the accumulator may hold at any point, so memory stays flat no
|
|
113094
|
-
// matter how many brace groups are chained. The limit sits well above any
|
|
113095
|
-
// realistic expansion (100k results hitting `EXPANSION_MAX` measure ~1M
|
|
113096
|
-
// characters) so legitimate input is unaffected.
|
|
113097
|
-
const EXPANSION_MAX_LENGTH = 4_000_000;
|
|
113098
|
-
function numeric(str) {
|
|
113099
|
-
return !isNaN(str) ? parseInt(str, 10) : str.charCodeAt(0);
|
|
113139
|
+
return hashTable2;
|
|
113140
|
+
}
|
|
113100
113141
|
}
|
|
113101
|
-
function
|
|
113102
|
-
|
|
113103
|
-
.replace(slashPattern, escSlash)
|
|
113104
|
-
.replace(openPattern, escOpen)
|
|
113105
|
-
.replace(closePattern, escClose)
|
|
113106
|
-
.replace(commaPattern, escComma)
|
|
113107
|
-
.replace(periodPattern, escPeriod);
|
|
113142
|
+
function makeBuffer(size) {
|
|
113143
|
+
return new Uint8Array(size);
|
|
113108
113144
|
}
|
|
113109
|
-
function
|
|
113110
|
-
|
|
113111
|
-
.replace(escSlashPattern, '\\')
|
|
113112
|
-
.replace(escOpenPattern, '{')
|
|
113113
|
-
.replace(escClosePattern, '}')
|
|
113114
|
-
.replace(escCommaPattern, ',')
|
|
113115
|
-
.replace(escPeriodPattern, '.');
|
|
113145
|
+
function sliceArray(array, start, end) {
|
|
113146
|
+
return array.slice(start, end);
|
|
113116
113147
|
}
|
|
113117
|
-
|
|
113118
|
-
|
|
113119
|
-
|
|
113120
|
-
|
|
113121
|
-
|
|
113122
|
-
|
|
113123
|
-
|
|
113124
|
-
|
|
113148
|
+
function decompressBound(src) {
|
|
113149
|
+
let sIndex = 0;
|
|
113150
|
+
if (readU32(src, sIndex) !== magicNum) {
|
|
113151
|
+
throw new Error("invalid magic number");
|
|
113152
|
+
}
|
|
113153
|
+
sIndex += 4;
|
|
113154
|
+
const descriptor = src[sIndex++];
|
|
113155
|
+
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
113156
|
+
throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask));
|
|
113157
|
+
}
|
|
113158
|
+
const useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
113159
|
+
const useContentSize = (descriptor & fdContentSize) !== 0;
|
|
113160
|
+
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
113161
|
+
if (bsMap[bsIdx] === void 0) {
|
|
113162
|
+
throw new Error("invalid block size " + bsIdx);
|
|
113163
|
+
}
|
|
113164
|
+
const maxBlockSize = bsMap[bsIdx];
|
|
113165
|
+
if (useContentSize) {
|
|
113166
|
+
return readU64(src, sIndex);
|
|
113167
|
+
}
|
|
113168
|
+
sIndex++;
|
|
113169
|
+
let maxSize = 0;
|
|
113170
|
+
while (true) {
|
|
113171
|
+
let blockSize = readU32(src, sIndex);
|
|
113172
|
+
sIndex += 4;
|
|
113173
|
+
if (blockSize & bsUncompressed) {
|
|
113174
|
+
blockSize &= ~bsUncompressed;
|
|
113175
|
+
maxSize += blockSize;
|
|
113176
|
+
} else if (blockSize > 0) {
|
|
113177
|
+
maxSize += maxBlockSize;
|
|
113125
113178
|
}
|
|
113126
|
-
|
|
113127
|
-
|
|
113128
|
-
if (!m) {
|
|
113129
|
-
return str.split(',');
|
|
113179
|
+
if (blockSize === 0) {
|
|
113180
|
+
return maxSize;
|
|
113130
113181
|
}
|
|
113131
|
-
|
|
113132
|
-
|
|
113133
|
-
p[p.length - 1] += '{' + body + '}';
|
|
113134
|
-
const postParts = parseCommaParts(post);
|
|
113135
|
-
if (post.length) {
|
|
113136
|
-
p[p.length - 1] += postParts.shift();
|
|
113137
|
-
p.push.apply(p, postParts);
|
|
113182
|
+
if (useBlockSum) {
|
|
113183
|
+
sIndex += 4;
|
|
113138
113184
|
}
|
|
113139
|
-
|
|
113140
|
-
|
|
113185
|
+
sIndex += blockSize;
|
|
113186
|
+
}
|
|
113141
113187
|
}
|
|
113142
|
-
function
|
|
113143
|
-
|
|
113144
|
-
|
|
113188
|
+
function decompressBlock(src, dst, sIndex, sLength, dIndex) {
|
|
113189
|
+
let mLength, mOffset, sEnd, n, i;
|
|
113190
|
+
const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0;
|
|
113191
|
+
sEnd = sIndex + sLength;
|
|
113192
|
+
while (sIndex < sEnd) {
|
|
113193
|
+
const token = src[sIndex++];
|
|
113194
|
+
let literalCount = token >> 4;
|
|
113195
|
+
if (literalCount > 0) {
|
|
113196
|
+
if (literalCount === 15) {
|
|
113197
|
+
while (true) {
|
|
113198
|
+
literalCount += src[sIndex];
|
|
113199
|
+
if (src[sIndex++] !== 255) {
|
|
113200
|
+
break;
|
|
113201
|
+
}
|
|
113202
|
+
}
|
|
113203
|
+
}
|
|
113204
|
+
for (n = sIndex + literalCount; sIndex < n; ) {
|
|
113205
|
+
dst[dIndex++] = src[sIndex++];
|
|
113206
|
+
}
|
|
113145
113207
|
}
|
|
113146
|
-
|
|
113147
|
-
|
|
113148
|
-
// Anything starting with {} will have the first two bytes preserved
|
|
113149
|
-
// but *only* at the top level, so {},a}b will not expand to anything,
|
|
113150
|
-
// but a{},b}c will be expanded to [a}c,abc].
|
|
113151
|
-
// One could argue that this is a bug in Bash, but since the goal of
|
|
113152
|
-
// this module is to match Bash's rules, we escape a leading {}
|
|
113153
|
-
if (str.slice(0, 2) === '{}') {
|
|
113154
|
-
str = '\\{\\}' + str.slice(2);
|
|
113208
|
+
if (sIndex >= sEnd) {
|
|
113209
|
+
break;
|
|
113155
113210
|
}
|
|
113156
|
-
|
|
113157
|
-
|
|
113158
|
-
|
|
113159
|
-
|
|
113160
|
-
|
|
113161
|
-
|
|
113162
|
-
|
|
113163
|
-
}
|
|
113164
|
-
function lte(i, y) {
|
|
113165
|
-
return i <= y;
|
|
113166
|
-
}
|
|
113167
|
-
function gte(i, y) {
|
|
113168
|
-
return i >= y;
|
|
113169
|
-
}
|
|
113170
|
-
// Build `{ acc[a] + pre + values[v] }` for every combination, capping the
|
|
113171
|
-
// number of results at `max` and the total number of characters at `maxLength`.
|
|
113172
|
-
// This is the one place output grows, so bounding it here keeps the single
|
|
113173
|
-
// accumulator - and therefore memory - flat regardless of how many brace groups
|
|
113174
|
-
// are combined (CVE-2026-14257).
|
|
113175
|
-
function combine(acc, pre, values, max, maxLength, dropEmpties) {
|
|
113176
|
-
const out = [];
|
|
113177
|
-
let length = 0;
|
|
113178
|
-
for (let a = 0; a < acc.length; a++) {
|
|
113179
|
-
for (let v = 0; v < values.length; v++) {
|
|
113180
|
-
if (out.length >= max)
|
|
113181
|
-
return out;
|
|
113182
|
-
const expansion = acc[a] + pre + values[v];
|
|
113183
|
-
// Bash drops empty results at the top level. Skip them before they count
|
|
113184
|
-
// against `max`, so `max` bounds the number of *kept* results.
|
|
113185
|
-
if (dropEmpties && !expansion)
|
|
113186
|
-
continue;
|
|
113187
|
-
if (length + expansion.length > maxLength)
|
|
113188
|
-
return out;
|
|
113189
|
-
out.push(expansion);
|
|
113190
|
-
length += expansion.length;
|
|
113211
|
+
mLength = token & 15;
|
|
113212
|
+
mOffset = src[sIndex++] | src[sIndex++] << 8;
|
|
113213
|
+
if (mLength === 15) {
|
|
113214
|
+
while (true) {
|
|
113215
|
+
mLength += src[sIndex];
|
|
113216
|
+
if (src[sIndex++] !== 255) {
|
|
113217
|
+
break;
|
|
113191
113218
|
}
|
|
113219
|
+
}
|
|
113192
113220
|
}
|
|
113193
|
-
|
|
113221
|
+
mLength += minMatch;
|
|
113222
|
+
if (hasCopyWithin && mOffset === 1) {
|
|
113223
|
+
dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength);
|
|
113224
|
+
dIndex += mLength;
|
|
113225
|
+
} else if (hasCopyWithin && mOffset > mLength && mLength > 31) {
|
|
113226
|
+
dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength);
|
|
113227
|
+
dIndex += mLength;
|
|
113228
|
+
} else {
|
|
113229
|
+
for (i = dIndex - mOffset, n = i + mLength; i < n; ) {
|
|
113230
|
+
dst[dIndex++] = dst[i++] | 0;
|
|
113231
|
+
}
|
|
113232
|
+
}
|
|
113233
|
+
}
|
|
113234
|
+
return dIndex;
|
|
113194
113235
|
}
|
|
113195
|
-
|
|
113196
|
-
|
|
113197
|
-
|
|
113198
|
-
|
|
113199
|
-
|
|
113200
|
-
|
|
113201
|
-
|
|
113202
|
-
|
|
113203
|
-
|
|
113204
|
-
|
|
113236
|
+
function decompressFrame(src, dst) {
|
|
113237
|
+
let useBlockSum, useContentSum, useContentSize, descriptor;
|
|
113238
|
+
let sIndex = 0;
|
|
113239
|
+
let dIndex = 0;
|
|
113240
|
+
if (readU32(src, sIndex) !== magicNum) {
|
|
113241
|
+
throw new Error("invalid magic number");
|
|
113242
|
+
}
|
|
113243
|
+
sIndex += 4;
|
|
113244
|
+
descriptor = src[sIndex++];
|
|
113245
|
+
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
113246
|
+
throw new Error("incompatible descriptor version");
|
|
113247
|
+
}
|
|
113248
|
+
useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
113249
|
+
useContentSum = (descriptor & fdContentChksum) !== 0;
|
|
113250
|
+
useContentSize = (descriptor & fdContentSize) !== 0;
|
|
113251
|
+
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
113252
|
+
if (bsMap[bsIdx] === void 0) {
|
|
113253
|
+
throw new Error("invalid block size");
|
|
113254
|
+
}
|
|
113255
|
+
if (useContentSize) {
|
|
113256
|
+
sIndex += 8;
|
|
113257
|
+
}
|
|
113258
|
+
sIndex++;
|
|
113259
|
+
while (true) {
|
|
113260
|
+
var compSize;
|
|
113261
|
+
compSize = readU32(src, sIndex);
|
|
113262
|
+
sIndex += 4;
|
|
113263
|
+
if (compSize === 0) {
|
|
113264
|
+
break;
|
|
113205
113265
|
}
|
|
113206
|
-
|
|
113207
|
-
|
|
113208
|
-
const y = numeric(n[1]);
|
|
113209
|
-
const width = Math.max(n[0].length, n[1].length);
|
|
113210
|
-
let incr = n.length === 3 && n[2] !== undefined ?
|
|
113211
|
-
Math.max(Math.abs(numeric(n[2])), 1)
|
|
113212
|
-
: 1;
|
|
113213
|
-
let test = lte;
|
|
113214
|
-
const reverse = y < x;
|
|
113215
|
-
if (reverse) {
|
|
113216
|
-
incr *= -1;
|
|
113217
|
-
test = gte;
|
|
113266
|
+
if (useBlockSum) {
|
|
113267
|
+
sIndex += 4;
|
|
113218
113268
|
}
|
|
113219
|
-
|
|
113220
|
-
|
|
113221
|
-
|
|
113222
|
-
|
|
113223
|
-
|
|
113224
|
-
|
|
113225
|
-
|
|
113226
|
-
|
|
113227
|
-
}
|
|
113228
|
-
}
|
|
113229
|
-
else {
|
|
113230
|
-
c = String(i);
|
|
113231
|
-
if (pad) {
|
|
113232
|
-
const need = width - c.length;
|
|
113233
|
-
if (need > 0) {
|
|
113234
|
-
const z = new Array(need + 1).join('0');
|
|
113235
|
-
if (i < 0) {
|
|
113236
|
-
c = '-' + z + c.slice(1);
|
|
113237
|
-
}
|
|
113238
|
-
else {
|
|
113239
|
-
c = z + c;
|
|
113240
|
-
}
|
|
113241
|
-
}
|
|
113242
|
-
}
|
|
113243
|
-
}
|
|
113244
|
-
if (length + c.length > maxLength)
|
|
113245
|
-
break;
|
|
113246
|
-
N.push(c);
|
|
113247
|
-
length += c.length;
|
|
113269
|
+
if ((compSize & bsUncompressed) !== 0) {
|
|
113270
|
+
compSize &= ~bsUncompressed;
|
|
113271
|
+
for (let j = 0; j < compSize; j++) {
|
|
113272
|
+
dst[dIndex++] = src[sIndex++];
|
|
113273
|
+
}
|
|
113274
|
+
} else {
|
|
113275
|
+
dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex);
|
|
113276
|
+
sIndex += compSize;
|
|
113248
113277
|
}
|
|
113249
|
-
|
|
113278
|
+
}
|
|
113279
|
+
if (useContentSum) {
|
|
113280
|
+
sIndex += 4;
|
|
113281
|
+
}
|
|
113282
|
+
return dIndex;
|
|
113250
113283
|
}
|
|
113251
|
-
function
|
|
113252
|
-
|
|
113253
|
-
|
|
113254
|
-
|
|
113255
|
-
|
|
113256
|
-
|
|
113257
|
-
|
|
113258
|
-
|
|
113259
|
-
|
|
113260
|
-
|
|
113261
|
-
|
|
113262
|
-
// them (the one with no brace set left in the tail).
|
|
113263
|
-
let dropEmpties = false;
|
|
113264
|
-
let firstGroup = true;
|
|
113265
|
-
for (;;) {
|
|
113266
|
-
const m = balanced('{', '}', str);
|
|
113267
|
-
// No brace set left: the rest of the string is literal.
|
|
113268
|
-
if (!m) {
|
|
113269
|
-
return combine(acc, str, [''], max, maxLength, dropEmpties);
|
|
113270
|
-
}
|
|
113271
|
-
// no need to expand pre, since it is guaranteed to be free of brace-sets
|
|
113272
|
-
const pre = m.pre;
|
|
113273
|
-
if (/\$$/.test(pre)) {
|
|
113274
|
-
acc = combine(acc, pre + '{' + m.body + '}', [''], max, maxLength, dropEmpties && !m.post.length);
|
|
113275
|
-
firstGroup = false;
|
|
113276
|
-
if (!m.post.length)
|
|
113277
|
-
break;
|
|
113278
|
-
str = m.post;
|
|
113279
|
-
continue;
|
|
113280
|
-
}
|
|
113281
|
-
const isNumericSequence = /^-?\d+\.\.-?\d+(?:\.\.-?\d+)?$/.test(m.body);
|
|
113282
|
-
const isAlphaSequence = /^[a-zA-Z]\.\.[a-zA-Z](?:\.\.-?\d+)?$/.test(m.body);
|
|
113283
|
-
const isSequence = isNumericSequence || isAlphaSequence;
|
|
113284
|
-
const isOptions = m.body.indexOf(',') >= 0;
|
|
113285
|
-
if (!isSequence && !isOptions) {
|
|
113286
|
-
// {a},b}
|
|
113287
|
-
if (m.post.match(/,(?!,).*\}/)) {
|
|
113288
|
-
str = m.pre + '{' + m.body + escClose + m.post;
|
|
113289
|
-
isTop = true;
|
|
113290
|
-
continue;
|
|
113291
|
-
}
|
|
113292
|
-
// Nothing here expands, so the whole remaining string is literal.
|
|
113293
|
-
return combine(acc, pre + '{' + m.body + '}' + m.post, [''], max, maxLength, dropEmpties);
|
|
113294
|
-
}
|
|
113295
|
-
if (firstGroup) {
|
|
113296
|
-
dropEmpties = isTop && !isSequence;
|
|
113297
|
-
firstGroup = false;
|
|
113298
|
-
}
|
|
113299
|
-
let values;
|
|
113300
|
-
if (isSequence) {
|
|
113301
|
-
values = expandSequence(m.body, isAlphaSequence, max, maxLength);
|
|
113302
|
-
}
|
|
113303
|
-
else {
|
|
113304
|
-
let n = parseCommaParts(m.body);
|
|
113305
|
-
if (n.length === 1 && n[0] !== undefined) {
|
|
113306
|
-
// x{{a,b}}y ==> x{a}y x{b}y
|
|
113307
|
-
n = expand_(n[0], max, maxLength, false).map(embrace);
|
|
113308
|
-
//XXX is this necessary? Can't seem to hit it in tests.
|
|
113309
|
-
/* c8 ignore start */
|
|
113310
|
-
if (n.length === 1) {
|
|
113311
|
-
acc = combine(acc, pre + n[0], [''], max, maxLength, dropEmpties && !m.post.length);
|
|
113312
|
-
if (!m.post.length)
|
|
113313
|
-
break;
|
|
113314
|
-
str = m.post;
|
|
113315
|
-
continue;
|
|
113316
|
-
}
|
|
113317
|
-
/* c8 ignore stop */
|
|
113318
|
-
}
|
|
113319
|
-
// Values that `combine` is going to drop as empty produce no result, so
|
|
113320
|
-
// they must not count against `max` - otherwise `{a,,b}` with `max: 2`
|
|
113321
|
-
// would stop at `['a', '']` and yield one result instead of two. Skipping
|
|
113322
|
-
// them outright keeps `values` bounded while leaving `max` a bound on
|
|
113323
|
-
// *kept* results.
|
|
113324
|
-
let dropsEmpties = dropEmpties && !m.post.length && !pre;
|
|
113325
|
-
for (let d = 0; dropsEmpties && d < acc.length; d++) {
|
|
113326
|
-
if (acc[d]) {
|
|
113327
|
-
dropsEmpties = false;
|
|
113328
|
-
}
|
|
113329
|
-
}
|
|
113330
|
-
values = [];
|
|
113331
|
-
let valuesLength = 0;
|
|
113332
|
-
outer: for (let j = 0; j < n.length; j++) {
|
|
113333
|
-
const expanded = expand_(n[j], max, maxLength, false);
|
|
113334
|
-
for (let k = 0; k < expanded.length; k++) {
|
|
113335
|
-
const v = expanded[k];
|
|
113336
|
-
if (dropsEmpties && !v)
|
|
113337
|
-
continue;
|
|
113338
|
-
if (values.length >= max || valuesLength + v.length > maxLength) {
|
|
113339
|
-
break outer;
|
|
113340
|
-
}
|
|
113341
|
-
values.push(v);
|
|
113342
|
-
valuesLength += v.length;
|
|
113343
|
-
}
|
|
113344
|
-
}
|
|
113345
|
-
}
|
|
113346
|
-
acc = combine(acc, pre, values, max, maxLength, dropEmpties && !m.post.length);
|
|
113347
|
-
if (!m.post.length)
|
|
113348
|
-
break;
|
|
113349
|
-
str = m.post;
|
|
113350
|
-
}
|
|
113351
|
-
return acc;
|
|
113284
|
+
function decompress(src, maxSize) {
|
|
113285
|
+
let dst, size;
|
|
113286
|
+
if (maxSize === void 0) {
|
|
113287
|
+
maxSize = decompressBound(src);
|
|
113288
|
+
}
|
|
113289
|
+
dst = makeBuffer(maxSize);
|
|
113290
|
+
size = decompressFrame(src, dst);
|
|
113291
|
+
if (size !== maxSize) {
|
|
113292
|
+
dst = sliceArray(dst, 0, size);
|
|
113293
|
+
}
|
|
113294
|
+
return dst;
|
|
113352
113295
|
}
|
|
113353
113296
|
|
|
113354
|
-
|
|
113355
|
-
|
|
113356
|
-
|
|
113357
|
-
|
|
113297
|
+
// src/utils/RangeList.ts
|
|
113298
|
+
var RangeList = class {
|
|
113299
|
+
ranges = [];
|
|
113300
|
+
/**
|
|
113301
|
+
* Add a range to the list. If it overlaps with existing ranges,
|
|
113302
|
+
* it will split them and increment reference counts accordingly.
|
|
113303
|
+
*/
|
|
113304
|
+
add(start, end) {
|
|
113305
|
+
if (end <= start) {
|
|
113306
|
+
throw new TypeError("End must be greater than start");
|
|
113358
113307
|
}
|
|
113359
|
-
|
|
113360
|
-
|
|
113308
|
+
const overlappingRanges = [];
|
|
113309
|
+
for (let i = 0; i < this.ranges.length; i++) {
|
|
113310
|
+
const range2 = this.ranges[i];
|
|
113311
|
+
if (start < range2.end && end > range2.start) {
|
|
113312
|
+
overlappingRanges.push({ index: i, range: range2 });
|
|
113313
|
+
}
|
|
113314
|
+
if (range2.data !== null) {
|
|
113315
|
+
throw new Error("Overlapping range already has data");
|
|
113316
|
+
}
|
|
113361
113317
|
}
|
|
113362
|
-
|
|
113363
|
-
|
|
113364
|
-
|
|
113365
|
-
|
|
113366
|
-
// { <posix class>: [<translation>, /u flag required, negated]
|
|
113367
|
-
const posixClasses = {
|
|
113368
|
-
'[:alnum:]': ['\\p{L}\\p{Nl}\\p{Nd}', true],
|
|
113369
|
-
'[:alpha:]': ['\\p{L}\\p{Nl}', true],
|
|
113370
|
-
'[:ascii:]': ['\\x' + '00-\\x' + '7f', false],
|
|
113371
|
-
'[:blank:]': ['\\p{Zs}\\t', true],
|
|
113372
|
-
'[:cntrl:]': ['\\p{Cc}', true],
|
|
113373
|
-
'[:digit:]': ['\\p{Nd}', true],
|
|
113374
|
-
'[:graph:]': ['\\p{Z}\\p{C}', true, true],
|
|
113375
|
-
'[:lower:]': ['\\p{Ll}', true],
|
|
113376
|
-
'[:print:]': ['\\p{C}', true],
|
|
113377
|
-
'[:punct:]': ['\\p{P}', true],
|
|
113378
|
-
'[:space:]': ['\\p{Z}\\t\\r\\n\\v\\f', true],
|
|
113379
|
-
'[:upper:]': ['\\p{Lu}', true],
|
|
113380
|
-
'[:word:]': ['\\p{L}\\p{Nl}\\p{Nd}\\p{Pc}', true],
|
|
113381
|
-
'[:xdigit:]': ['A-Fa-f0-9', false],
|
|
113382
|
-
};
|
|
113383
|
-
// only need to escape a few things inside of brace expressions
|
|
113384
|
-
// escapes: [ \ ] -
|
|
113385
|
-
const braceEscape = (s) => s.replace(/[[\]\\-]/g, '\\$&');
|
|
113386
|
-
// escape all regexp magic characters
|
|
113387
|
-
const regexpEscape = (s) => s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&');
|
|
113388
|
-
// everything has already been escaped, we just have to join
|
|
113389
|
-
const rangesToString = (ranges) => ranges.join('');
|
|
113390
|
-
// takes a glob string at a posix brace expression, and returns
|
|
113391
|
-
// an equivalent regular expression source, and boolean indicating
|
|
113392
|
-
// whether the /u flag needs to be applied, and the number of chars
|
|
113393
|
-
// consumed to parse the character class.
|
|
113394
|
-
// This also removes out of order ranges, and returns ($.) if the
|
|
113395
|
-
// entire class just no good.
|
|
113396
|
-
const parseClass = (glob, position) => {
|
|
113397
|
-
const pos = position;
|
|
113398
|
-
/* c8 ignore start */
|
|
113399
|
-
if (glob.charAt(pos) !== '[') {
|
|
113400
|
-
throw new Error('not in a brace expression');
|
|
113318
|
+
if (overlappingRanges.length === 0) {
|
|
113319
|
+
this.ranges.push({ start, end, refCount: 1, data: null });
|
|
113320
|
+
this.ranges.sort((a, b) => a.start - b.start);
|
|
113321
|
+
return;
|
|
113401
113322
|
}
|
|
113402
|
-
|
|
113403
|
-
|
|
113404
|
-
|
|
113405
|
-
|
|
113406
|
-
|
|
113407
|
-
|
|
113408
|
-
|
|
113409
|
-
|
|
113410
|
-
|
|
113411
|
-
|
|
113412
|
-
|
|
113413
|
-
|
|
113414
|
-
|
|
113415
|
-
|
|
113416
|
-
|
|
113417
|
-
|
|
113418
|
-
|
|
113419
|
-
|
|
113420
|
-
|
|
113421
|
-
|
|
113422
|
-
|
|
113423
|
-
|
|
113424
|
-
|
|
113425
|
-
|
|
113426
|
-
|
|
113427
|
-
|
|
113428
|
-
|
|
113429
|
-
|
|
113430
|
-
|
|
113431
|
-
|
|
113432
|
-
|
|
113433
|
-
|
|
113434
|
-
|
|
113435
|
-
|
|
113436
|
-
// invalid, [a-[] is fine, but not [a-[:alpha]]
|
|
113437
|
-
if (rangeStart) {
|
|
113438
|
-
return ['$.', false, glob.length - pos, true];
|
|
113439
|
-
}
|
|
113440
|
-
i += cls.length;
|
|
113441
|
-
if (neg)
|
|
113442
|
-
negs.push(unip);
|
|
113443
|
-
else
|
|
113444
|
-
ranges.push(unip);
|
|
113445
|
-
uflag = uflag || u;
|
|
113446
|
-
continue WHILE;
|
|
113447
|
-
}
|
|
113448
|
-
}
|
|
113449
|
-
}
|
|
113450
|
-
// now it's just a normal character, effectively
|
|
113451
|
-
escaping = false;
|
|
113452
|
-
if (rangeStart) {
|
|
113453
|
-
// throw this range away if it's not valid, but others
|
|
113454
|
-
// can still match.
|
|
113455
|
-
if (c > rangeStart) {
|
|
113456
|
-
ranges.push(braceEscape(rangeStart) + '-' + braceEscape(c));
|
|
113457
|
-
}
|
|
113458
|
-
else if (c === rangeStart) {
|
|
113459
|
-
ranges.push(braceEscape(c));
|
|
113460
|
-
}
|
|
113461
|
-
rangeStart = '';
|
|
113462
|
-
i++;
|
|
113463
|
-
continue;
|
|
113464
|
-
}
|
|
113465
|
-
// now might be the start of a range.
|
|
113466
|
-
// can be either c-d or c-] or c<more...>] or c] at this point
|
|
113467
|
-
if (glob.startsWith('-]', i + 1)) {
|
|
113468
|
-
ranges.push(braceEscape(c + '-'));
|
|
113469
|
-
i += 2;
|
|
113470
|
-
continue;
|
|
113471
|
-
}
|
|
113472
|
-
if (glob.startsWith('-', i + 1)) {
|
|
113473
|
-
rangeStart = c;
|
|
113474
|
-
i += 2;
|
|
113475
|
-
continue;
|
|
113476
|
-
}
|
|
113477
|
-
// not the start of a range, just a single character
|
|
113478
|
-
ranges.push(braceEscape(c));
|
|
113479
|
-
i++;
|
|
113323
|
+
const newRanges = [];
|
|
113324
|
+
let currentPos = start;
|
|
113325
|
+
for (let i = 0; i < overlappingRanges.length; i++) {
|
|
113326
|
+
const { range: range2 } = overlappingRanges[i];
|
|
113327
|
+
if (currentPos < range2.start) {
|
|
113328
|
+
newRanges.push({
|
|
113329
|
+
start: currentPos,
|
|
113330
|
+
end: range2.start,
|
|
113331
|
+
refCount: 1,
|
|
113332
|
+
data: null
|
|
113333
|
+
});
|
|
113334
|
+
} else if (range2.start < currentPos) {
|
|
113335
|
+
newRanges.push({
|
|
113336
|
+
start: range2.start,
|
|
113337
|
+
end: currentPos,
|
|
113338
|
+
refCount: range2.refCount,
|
|
113339
|
+
data: null
|
|
113340
|
+
});
|
|
113341
|
+
}
|
|
113342
|
+
newRanges.push({
|
|
113343
|
+
start: Math.max(currentPos, range2.start),
|
|
113344
|
+
end: Math.min(end, range2.end),
|
|
113345
|
+
refCount: range2.refCount + 1,
|
|
113346
|
+
data: null
|
|
113347
|
+
});
|
|
113348
|
+
if (range2.end > end) {
|
|
113349
|
+
newRanges.push({
|
|
113350
|
+
start: end,
|
|
113351
|
+
end: range2.end,
|
|
113352
|
+
refCount: range2.refCount,
|
|
113353
|
+
data: null
|
|
113354
|
+
});
|
|
113355
|
+
}
|
|
113356
|
+
currentPos = Math.max(currentPos, range2.end);
|
|
113480
113357
|
}
|
|
113481
|
-
if (
|
|
113482
|
-
|
|
113483
|
-
|
|
113484
|
-
|
|
113358
|
+
if (currentPos < end) {
|
|
113359
|
+
newRanges.push({
|
|
113360
|
+
start: currentPos,
|
|
113361
|
+
end,
|
|
113362
|
+
refCount: 1,
|
|
113363
|
+
data: null
|
|
113364
|
+
});
|
|
113485
113365
|
}
|
|
113486
|
-
|
|
113487
|
-
|
|
113488
|
-
|
|
113489
|
-
|
|
113366
|
+
const firstIndex = overlappingRanges[0].index;
|
|
113367
|
+
const lastIndex = overlappingRanges[overlappingRanges.length - 1].index;
|
|
113368
|
+
this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges);
|
|
113369
|
+
this.ranges.sort((a, b) => a.start - b.start);
|
|
113370
|
+
}
|
|
113371
|
+
/**
|
|
113372
|
+
* Remove a range from the list. The range must start and end at existing boundaries.
|
|
113373
|
+
*/
|
|
113374
|
+
remove(start, end) {
|
|
113375
|
+
if (end <= start) {
|
|
113376
|
+
throw new TypeError("End must be greater than start");
|
|
113490
113377
|
}
|
|
113491
|
-
|
|
113492
|
-
|
|
113493
|
-
|
|
113494
|
-
|
|
113495
|
-
|
|
113496
|
-
|
|
113497
|
-
/^\\?.$/.test(ranges[0]) &&
|
|
113498
|
-
!negate) {
|
|
113499
|
-
const r = ranges[0].length === 2 ? ranges[0].slice(-1) : ranges[0];
|
|
113500
|
-
return [regexpEscape(r), false, endPos - pos, false];
|
|
113378
|
+
const affectedRanges = [];
|
|
113379
|
+
for (let i = 0; i < this.ranges.length; i++) {
|
|
113380
|
+
const range2 = this.ranges[i];
|
|
113381
|
+
if (start < range2.end && end > range2.start) {
|
|
113382
|
+
affectedRanges.push({ index: i, range: range2 });
|
|
113383
|
+
}
|
|
113501
113384
|
}
|
|
113502
|
-
|
|
113503
|
-
|
|
113504
|
-
const comb = ranges.length && negs.length ? '(' + sranges + '|' + snegs + ')'
|
|
113505
|
-
: ranges.length ? sranges
|
|
113506
|
-
: snegs;
|
|
113507
|
-
return [comb, uflag, endPos - pos, true];
|
|
113508
|
-
};
|
|
113509
|
-
|
|
113510
|
-
/**
|
|
113511
|
-
* Un-escape a string that has been escaped with {@link escape}.
|
|
113512
|
-
*
|
|
113513
|
-
* If the {@link MinimatchOptions.windowsPathsNoEscape} option is used, then
|
|
113514
|
-
* square-bracket escapes are removed, but not backslash escapes.
|
|
113515
|
-
*
|
|
113516
|
-
* For example, it will turn the string `'[*]'` into `*`, but it will not
|
|
113517
|
-
* turn `'\\*'` into `'*'`, because `\` is a path separator in
|
|
113518
|
-
* `windowsPathsNoEscape` mode.
|
|
113519
|
-
*
|
|
113520
|
-
* When `windowsPathsNoEscape` is not set, then both square-bracket escapes and
|
|
113521
|
-
* backslash escapes are removed.
|
|
113522
|
-
*
|
|
113523
|
-
* Slashes (and backslashes in `windowsPathsNoEscape` mode) cannot be escaped
|
|
113524
|
-
* or unescaped.
|
|
113525
|
-
*
|
|
113526
|
-
* When `magicalBraces` is not set, escapes of braces (`{` and `}`) will not be
|
|
113527
|
-
* unescaped.
|
|
113528
|
-
*/
|
|
113529
|
-
const unescape$1 = (s, { windowsPathsNoEscape = false, magicalBraces = true, } = {}) => {
|
|
113530
|
-
if (magicalBraces) {
|
|
113531
|
-
return windowsPathsNoEscape ?
|
|
113532
|
-
s.replace(/\[([^/\\])\]/g, '$1')
|
|
113533
|
-
: s
|
|
113534
|
-
.replace(/((?!\\).|^)\[([^/\\])\]/g, '$1$2')
|
|
113535
|
-
.replace(/\\([^/])/g, '$1');
|
|
113385
|
+
if (affectedRanges.length === 0) {
|
|
113386
|
+
throw new Error("No ranges found to remove");
|
|
113536
113387
|
}
|
|
113537
|
-
|
|
113538
|
-
|
|
113539
|
-
|
|
113540
|
-
|
|
113541
|
-
|
|
113388
|
+
if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) {
|
|
113389
|
+
throw new Error("Range boundaries must match existing boundaries");
|
|
113390
|
+
}
|
|
113391
|
+
for (let i = 0; i < affectedRanges.length; i++) {
|
|
113392
|
+
const { range: range2 } = affectedRanges[i];
|
|
113393
|
+
range2.refCount--;
|
|
113394
|
+
}
|
|
113395
|
+
this.ranges = this.ranges.filter((range2) => range2.refCount > 0);
|
|
113396
|
+
}
|
|
113397
|
+
/**
|
|
113398
|
+
* Get all ranges within the specified boundaries.
|
|
113399
|
+
*/
|
|
113400
|
+
getRanges(start, end) {
|
|
113401
|
+
if (end <= start) {
|
|
113402
|
+
throw new TypeError("End must be greater than start");
|
|
113403
|
+
}
|
|
113404
|
+
return this.ranges.filter((range2) => start < range2.end && end > range2.start);
|
|
113405
|
+
}
|
|
113406
|
+
/**
|
|
113407
|
+
* Get all ranges in the list
|
|
113408
|
+
*/
|
|
113409
|
+
getAllRanges() {
|
|
113410
|
+
return [...this.ranges];
|
|
113411
|
+
}
|
|
113542
113412
|
};
|
|
113543
113413
|
|
|
113544
|
-
//
|
|
113545
|
-
var
|
|
113546
|
-
|
|
113547
|
-
|
|
113548
|
-
|
|
113549
|
-
|
|
113550
|
-
|
|
113551
|
-
|
|
113552
|
-
|
|
113553
|
-
|
|
113554
|
-
|
|
113555
|
-
|
|
113556
|
-
|
|
113557
|
-
|
|
113558
|
-
|
|
113559
|
-
|
|
113560
|
-
|
|
113561
|
-
|
|
113562
|
-
|
|
113563
|
-
|
|
113564
|
-
|
|
113565
|
-
|
|
113566
|
-
|
|
113567
|
-
|
|
113568
|
-
|
|
113569
|
-
|
|
113570
|
-
|
|
113571
|
-
|
|
113572
|
-
|
|
113573
|
-
|
|
113574
|
-
|
|
113575
|
-
|
|
113576
|
-
|
|
113577
|
-
|
|
113578
|
-
|
|
113579
|
-
|
|
113580
|
-
|
|
113581
|
-
|
|
113414
|
+
// src/utils/XetBlob.ts
|
|
113415
|
+
var JWT_SAFETY_PERIOD = 6e4;
|
|
113416
|
+
var JWT_CACHE_SIZE = 1e3;
|
|
113417
|
+
var compressionSchemeLabels = {
|
|
113418
|
+
[0 /* None */]: "None",
|
|
113419
|
+
[1 /* LZ4 */]: "LZ4",
|
|
113420
|
+
[2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4"
|
|
113421
|
+
};
|
|
113422
|
+
var XET_CHUNK_HEADER_BYTES = 8;
|
|
113423
|
+
var XetBlob = class extends Blob {
|
|
113424
|
+
fetch;
|
|
113425
|
+
accessToken;
|
|
113426
|
+
refreshUrl;
|
|
113427
|
+
reconstructionUrl;
|
|
113428
|
+
hash;
|
|
113429
|
+
start = 0;
|
|
113430
|
+
end = 0;
|
|
113431
|
+
internalLogging = false;
|
|
113432
|
+
reconstructionInfo;
|
|
113433
|
+
listener;
|
|
113434
|
+
constructor(params) {
|
|
113435
|
+
super([]);
|
|
113436
|
+
this.fetch = params.fetch ?? fetch.bind(globalThis);
|
|
113437
|
+
this.accessToken = checkCredentials(params);
|
|
113438
|
+
this.refreshUrl = params.refreshUrl;
|
|
113439
|
+
this.end = params.size;
|
|
113440
|
+
this.reconstructionUrl = params.reconstructionUrl;
|
|
113441
|
+
this.hash = params.hash;
|
|
113442
|
+
this.listener = params.listener;
|
|
113443
|
+
this.internalLogging = params.internalLogging ?? false;
|
|
113444
|
+
this.refreshUrl;
|
|
113445
|
+
}
|
|
113446
|
+
get size() {
|
|
113447
|
+
return this.end - this.start;
|
|
113448
|
+
}
|
|
113449
|
+
#clone() {
|
|
113450
|
+
const blob = new XetBlob({
|
|
113451
|
+
fetch: this.fetch,
|
|
113452
|
+
hash: this.hash,
|
|
113453
|
+
refreshUrl: this.refreshUrl,
|
|
113454
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
113455
|
+
reconstructionUrl: this.reconstructionUrl,
|
|
113456
|
+
size: this.size
|
|
113457
|
+
});
|
|
113458
|
+
blob.accessToken = this.accessToken;
|
|
113459
|
+
blob.start = this.start;
|
|
113460
|
+
blob.end = this.end;
|
|
113461
|
+
blob.reconstructionInfo = this.reconstructionInfo;
|
|
113462
|
+
blob.listener = this.listener;
|
|
113463
|
+
blob.internalLogging = this.internalLogging;
|
|
113464
|
+
return blob;
|
|
113465
|
+
}
|
|
113466
|
+
slice(start = 0, end = this.size) {
|
|
113467
|
+
const slice = this.#clone();
|
|
113468
|
+
slice.start = this.start + start;
|
|
113469
|
+
slice.end = Math.min(this.start + end, this.end);
|
|
113470
|
+
if (slice.start !== this.start || slice.end !== this.end) {
|
|
113471
|
+
slice.reconstructionInfo = void 0;
|
|
113472
|
+
}
|
|
113473
|
+
return slice;
|
|
113474
|
+
}
|
|
113475
|
+
#reconstructionInfoPromise;
|
|
113476
|
+
#loadReconstructionInfo() {
|
|
113477
|
+
if (this.#reconstructionInfoPromise) {
|
|
113478
|
+
return this.#reconstructionInfoPromise;
|
|
113479
|
+
}
|
|
113480
|
+
this.#reconstructionInfoPromise = (async () => {
|
|
113481
|
+
const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl);
|
|
113482
|
+
const resp = await this.fetch(this.reconstructionUrl ?? `${connParams.casUrl}/v1/reconstructions/${this.hash}`, {
|
|
113483
|
+
headers: {
|
|
113484
|
+
Authorization: `Bearer ${connParams.accessToken}`,
|
|
113485
|
+
Range: `bytes=${this.start}-${this.end - 1}`
|
|
113486
|
+
}
|
|
113487
|
+
});
|
|
113488
|
+
if (!resp.ok) {
|
|
113489
|
+
throw await createApiError(resp);
|
|
113490
|
+
}
|
|
113491
|
+
this.reconstructionInfo = await resp.json();
|
|
113492
|
+
return this.reconstructionInfo;
|
|
113493
|
+
})().finally(() => this.#reconstructionInfoPromise = void 0);
|
|
113494
|
+
return this.#reconstructionInfoPromise;
|
|
113495
|
+
}
|
|
113496
|
+
async #fetch() {
|
|
113497
|
+
if (!this.reconstructionInfo) {
|
|
113498
|
+
await this.#loadReconstructionInfo();
|
|
113499
|
+
}
|
|
113500
|
+
const rangeLists = /* @__PURE__ */ new Map();
|
|
113501
|
+
if (!this.reconstructionInfo) {
|
|
113502
|
+
throw new Error("Failed to load reconstruction info");
|
|
113503
|
+
}
|
|
113504
|
+
for (const term of this.reconstructionInfo.terms) {
|
|
113505
|
+
let rangeList = rangeLists.get(term.hash);
|
|
113506
|
+
if (!rangeList) {
|
|
113507
|
+
rangeList = new RangeList();
|
|
113508
|
+
rangeLists.set(term.hash, rangeList);
|
|
113509
|
+
}
|
|
113510
|
+
rangeList.add(term.range.start, term.range.end);
|
|
113511
|
+
}
|
|
113512
|
+
const listener = this.listener;
|
|
113513
|
+
const log = this.internalLogging ? (...args) => console.log(...args) : () => {
|
|
113514
|
+
};
|
|
113515
|
+
async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) {
|
|
113516
|
+
let totalBytesRead = 0;
|
|
113517
|
+
let readBytesToSkip = reconstructionInfo.offset_into_first_range;
|
|
113518
|
+
for (const term of reconstructionInfo.terms) {
|
|
113519
|
+
if (totalBytesRead >= maxBytes) {
|
|
113520
|
+
break;
|
|
113521
|
+
}
|
|
113522
|
+
const rangeList = rangeLists.get(term.hash);
|
|
113523
|
+
if (!rangeList) {
|
|
113524
|
+
throw new Error(`Failed to find range list for term ${term.hash}`);
|
|
113525
|
+
}
|
|
113526
|
+
{
|
|
113527
|
+
const termRanges = rangeList.getRanges(term.range.start, term.range.end);
|
|
113528
|
+
if (termRanges.every((range2) => range2.data)) {
|
|
113529
|
+
log("all data available for term", term.hash, readBytesToSkip);
|
|
113530
|
+
rangeLoop:
|
|
113531
|
+
for (const range2 of termRanges) {
|
|
113532
|
+
for (let chunk2 of range2.data) {
|
|
113533
|
+
if (readBytesToSkip) {
|
|
113534
|
+
const skipped = Math.min(readBytesToSkip, chunk2.byteLength);
|
|
113535
|
+
chunk2 = chunk2.slice(skipped);
|
|
113536
|
+
readBytesToSkip -= skipped;
|
|
113537
|
+
if (!chunk2.byteLength) {
|
|
113538
|
+
continue;
|
|
113539
|
+
}
|
|
113540
|
+
}
|
|
113541
|
+
if (chunk2.byteLength > maxBytes - totalBytesRead) {
|
|
113542
|
+
chunk2 = chunk2.slice(0, maxBytes - totalBytesRead);
|
|
113543
|
+
}
|
|
113544
|
+
totalBytesRead += chunk2.byteLength;
|
|
113545
|
+
yield range2.refCount > 1 ? chunk2.slice() : chunk2;
|
|
113546
|
+
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
113547
|
+
if (totalBytesRead >= maxBytes) {
|
|
113548
|
+
break rangeLoop;
|
|
113549
|
+
}
|
|
113550
|
+
}
|
|
113551
|
+
}
|
|
113552
|
+
rangeList.remove(term.range.start, term.range.end);
|
|
113553
|
+
continue;
|
|
113554
|
+
}
|
|
113555
|
+
}
|
|
113556
|
+
const fetchInfo = reconstructionInfo.fetch_info[term.hash].find(
|
|
113557
|
+
(info) => info.range.start <= term.range.start && info.range.end >= term.range.end
|
|
113558
|
+
);
|
|
113559
|
+
if (!fetchInfo) {
|
|
113560
|
+
throw new Error(
|
|
113561
|
+
`Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}`
|
|
113562
|
+
);
|
|
113563
|
+
}
|
|
113564
|
+
log("term", term);
|
|
113565
|
+
log("fetchinfo", fetchInfo);
|
|
113566
|
+
log("readBytesToSkip", readBytesToSkip);
|
|
113567
|
+
let resp = await customFetch(fetchInfo.url, {
|
|
113568
|
+
headers: {
|
|
113569
|
+
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
113570
|
+
}
|
|
113571
|
+
});
|
|
113572
|
+
if (resp.status === 403) {
|
|
113573
|
+
reconstructionInfo = await reloadReconstructionInfo();
|
|
113574
|
+
resp = await customFetch(fetchInfo.url, {
|
|
113575
|
+
headers: {
|
|
113576
|
+
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
113577
|
+
}
|
|
113578
|
+
});
|
|
113579
|
+
}
|
|
113580
|
+
if (!resp.ok) {
|
|
113581
|
+
throw await createApiError(resp);
|
|
113582
|
+
}
|
|
113583
|
+
log(
|
|
113584
|
+
"expected content length",
|
|
113585
|
+
resp.headers.get("content-length"),
|
|
113586
|
+
"range",
|
|
113587
|
+
fetchInfo.url_range,
|
|
113588
|
+
resp.headers.get("content-range")
|
|
113589
|
+
);
|
|
113590
|
+
const reader = resp.body?.getReader();
|
|
113591
|
+
if (!reader) {
|
|
113592
|
+
throw new Error("Failed to get reader from response body");
|
|
113593
|
+
}
|
|
113594
|
+
let done = false;
|
|
113595
|
+
let chunkIndex = fetchInfo.range.start;
|
|
113596
|
+
const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end);
|
|
113597
|
+
let leftoverBytes = void 0;
|
|
113598
|
+
let totalFetchBytes = 0;
|
|
113599
|
+
fetchData:
|
|
113600
|
+
while (!done && totalBytesRead < maxBytes) {
|
|
113601
|
+
const result = await reader.read();
|
|
113602
|
+
listener?.({ event: "read" });
|
|
113603
|
+
done = result.done;
|
|
113604
|
+
log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip);
|
|
113605
|
+
if (!result.value) {
|
|
113606
|
+
log("no data in result, cancelled", result);
|
|
113607
|
+
continue;
|
|
113608
|
+
}
|
|
113609
|
+
totalFetchBytes += result.value.byteLength;
|
|
113610
|
+
if (leftoverBytes) {
|
|
113611
|
+
result.value = combineUint8Arrays(leftoverBytes, result.value);
|
|
113612
|
+
leftoverBytes = void 0;
|
|
113613
|
+
}
|
|
113614
|
+
while (totalBytesRead < maxBytes && result.value?.byteLength) {
|
|
113615
|
+
if (result.value.byteLength < 8) {
|
|
113616
|
+
leftoverBytes = result.value;
|
|
113617
|
+
continue fetchData;
|
|
113618
|
+
}
|
|
113619
|
+
const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES);
|
|
113620
|
+
const chunkHeader = {
|
|
113621
|
+
version: header.getUint8(0),
|
|
113622
|
+
compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16,
|
|
113623
|
+
compression_scheme: header.getUint8(4),
|
|
113624
|
+
uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16
|
|
113625
|
+
};
|
|
113626
|
+
log("chunk header", chunkHeader, "to skip", readBytesToSkip);
|
|
113627
|
+
if (chunkHeader.version !== 0) {
|
|
113628
|
+
throw new Error(`Unsupported chunk version ${chunkHeader.version}`);
|
|
113629
|
+
}
|
|
113630
|
+
if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) {
|
|
113631
|
+
throw new Error(
|
|
113632
|
+
`Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}`
|
|
113633
|
+
);
|
|
113634
|
+
}
|
|
113635
|
+
if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) {
|
|
113636
|
+
leftoverBytes = result.value;
|
|
113637
|
+
continue fetchData;
|
|
113638
|
+
}
|
|
113639
|
+
result.value = result.value.slice(XET_CHUNK_HEADER_BYTES);
|
|
113640
|
+
let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes(
|
|
113641
|
+
decompress(
|
|
113642
|
+
result.value.slice(0, chunkHeader.compressed_length),
|
|
113643
|
+
chunkHeader.uncompressed_length
|
|
113644
|
+
)
|
|
113645
|
+
) : result.value.slice(0, chunkHeader.compressed_length);
|
|
113646
|
+
const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end);
|
|
113647
|
+
const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end;
|
|
113648
|
+
const minRefCountToStore = shouldYield ? 2 : 1;
|
|
113649
|
+
let stored = false;
|
|
113650
|
+
if (range2 && range2.refCount >= minRefCountToStore) {
|
|
113651
|
+
range2.data ??= [];
|
|
113652
|
+
range2.data.push(uncompressed);
|
|
113653
|
+
stored = true;
|
|
113654
|
+
}
|
|
113655
|
+
if (shouldYield) {
|
|
113656
|
+
if (readBytesToSkip) {
|
|
113657
|
+
const skipped = Math.min(readBytesToSkip, uncompressed.byteLength);
|
|
113658
|
+
uncompressed = uncompressed.slice(readBytesToSkip);
|
|
113659
|
+
readBytesToSkip -= skipped;
|
|
113660
|
+
}
|
|
113661
|
+
if (uncompressed.byteLength > maxBytes - totalBytesRead) {
|
|
113662
|
+
uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead);
|
|
113663
|
+
}
|
|
113664
|
+
if (uncompressed.byteLength) {
|
|
113665
|
+
log(
|
|
113666
|
+
"yield",
|
|
113667
|
+
uncompressed.byteLength,
|
|
113668
|
+
"bytes",
|
|
113669
|
+
result.value.byteLength,
|
|
113670
|
+
"total read",
|
|
113671
|
+
totalBytesRead,
|
|
113672
|
+
stored
|
|
113673
|
+
);
|
|
113674
|
+
totalBytesRead += uncompressed.byteLength;
|
|
113675
|
+
yield stored ? uncompressed.slice() : uncompressed;
|
|
113676
|
+
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
113677
|
+
}
|
|
113678
|
+
}
|
|
113679
|
+
chunkIndex++;
|
|
113680
|
+
result.value = result.value.slice(chunkHeader.compressed_length);
|
|
113681
|
+
}
|
|
113682
|
+
}
|
|
113683
|
+
if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) {
|
|
113684
|
+
log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes);
|
|
113685
|
+
log("failed to fetch all data for term", term.hash);
|
|
113686
|
+
throw new Error(
|
|
113687
|
+
`Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}`
|
|
113688
|
+
);
|
|
113689
|
+
}
|
|
113690
|
+
log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes);
|
|
113691
|
+
log("cancel reader");
|
|
113692
|
+
await reader.cancel();
|
|
113693
|
+
}
|
|
113694
|
+
}
|
|
113695
|
+
const iterator = readData(
|
|
113696
|
+
this.reconstructionInfo,
|
|
113697
|
+
this.fetch,
|
|
113698
|
+
this.end - this.start,
|
|
113699
|
+
this.#loadReconstructionInfo.bind(this)
|
|
113700
|
+
);
|
|
113701
|
+
return new ReadableStream(
|
|
113702
|
+
{
|
|
113703
|
+
// todo: when Safari supports it, type controller as ReadableByteStreamController
|
|
113704
|
+
async pull(controller) {
|
|
113705
|
+
const result = await iterator.next();
|
|
113706
|
+
if (result.value) {
|
|
113707
|
+
controller.enqueue(result.value);
|
|
113708
|
+
}
|
|
113709
|
+
if (result.done) {
|
|
113710
|
+
controller.close();
|
|
113711
|
+
}
|
|
113712
|
+
},
|
|
113713
|
+
type: "bytes"
|
|
113714
|
+
// todo: when Safari supports it, add autoAllocateChunkSize param
|
|
113715
|
+
},
|
|
113716
|
+
// todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function
|
|
113717
|
+
{
|
|
113718
|
+
highWaterMark: 1e3
|
|
113719
|
+
// 1_000 chunks for ~1MB of RAM
|
|
113720
|
+
}
|
|
113721
|
+
);
|
|
113722
|
+
}
|
|
113723
|
+
async arrayBuffer() {
|
|
113724
|
+
const result = await this.#fetch();
|
|
113725
|
+
return new Response(result).arrayBuffer();
|
|
113726
|
+
}
|
|
113727
|
+
async text() {
|
|
113728
|
+
const result = await this.#fetch();
|
|
113729
|
+
return new Response(result).text();
|
|
113730
|
+
}
|
|
113731
|
+
async response() {
|
|
113732
|
+
const result = await this.#fetch();
|
|
113733
|
+
return new Response(result);
|
|
113734
|
+
}
|
|
113735
|
+
stream() {
|
|
113736
|
+
const stream = new TransformStream();
|
|
113737
|
+
this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
113738
|
+
return stream.readable;
|
|
113739
|
+
}
|
|
113740
|
+
};
|
|
113741
|
+
var jwtPromises = /* @__PURE__ */ new Map();
|
|
113742
|
+
var jwts = /* @__PURE__ */ new Map();
|
|
113743
|
+
function cacheKey(params) {
|
|
113744
|
+
return JSON.stringify([params.refreshUrl, params.initialAccessToken]);
|
|
113745
|
+
}
|
|
113746
|
+
function bg4_regroup_bytes(bytes) {
|
|
113747
|
+
const split = Math.floor(bytes.byteLength / 4);
|
|
113748
|
+
const rem = bytes.byteLength % 4;
|
|
113749
|
+
const g1_pos = split + (rem >= 1 ? 1 : 0);
|
|
113750
|
+
const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0);
|
|
113751
|
+
const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0);
|
|
113752
|
+
const ret = new Uint8Array(bytes.byteLength);
|
|
113753
|
+
for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) {
|
|
113754
|
+
ret[i] = bytes[j];
|
|
113755
|
+
}
|
|
113756
|
+
for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) {
|
|
113757
|
+
ret[i] = bytes[j];
|
|
113758
|
+
}
|
|
113759
|
+
for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) {
|
|
113760
|
+
ret[i] = bytes[j];
|
|
113761
|
+
}
|
|
113762
|
+
for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) {
|
|
113763
|
+
ret[i] = bytes[j];
|
|
113764
|
+
}
|
|
113765
|
+
return ret;
|
|
113766
|
+
}
|
|
113767
|
+
async function getAccessToken(initialAccessToken, customFetch, refreshUrl) {
|
|
113768
|
+
const key = cacheKey({ refreshUrl, initialAccessToken });
|
|
113769
|
+
const jwt = jwts.get(key);
|
|
113770
|
+
if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) {
|
|
113771
|
+
return { accessToken: jwt.accessToken, casUrl: jwt.casUrl };
|
|
113772
|
+
}
|
|
113773
|
+
const existingPromise = jwtPromises.get(key);
|
|
113774
|
+
if (existingPromise) {
|
|
113775
|
+
return existingPromise;
|
|
113776
|
+
}
|
|
113777
|
+
const promise = (async () => {
|
|
113778
|
+
const resp = await customFetch(refreshUrl, {
|
|
113779
|
+
headers: {
|
|
113780
|
+
...initialAccessToken ? {
|
|
113781
|
+
Authorization: `Bearer ${initialAccessToken}`
|
|
113782
|
+
} : {}
|
|
113783
|
+
}
|
|
113784
|
+
});
|
|
113785
|
+
if (!resp.ok) {
|
|
113786
|
+
throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`);
|
|
113787
|
+
}
|
|
113788
|
+
const json = await resp.json();
|
|
113789
|
+
const jwt2 = {
|
|
113790
|
+
accessToken: json.accessToken,
|
|
113791
|
+
expiresAt: new Date(json.exp * 1e3),
|
|
113792
|
+
casUrl: json.casUrl
|
|
113793
|
+
};
|
|
113794
|
+
jwtPromises.delete(key);
|
|
113795
|
+
for (const [key2, value] of jwts.entries()) {
|
|
113796
|
+
if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) {
|
|
113797
|
+
jwts.delete(key2);
|
|
113798
|
+
} else {
|
|
113799
|
+
break;
|
|
113800
|
+
}
|
|
113801
|
+
}
|
|
113802
|
+
if (jwts.size >= JWT_CACHE_SIZE) {
|
|
113803
|
+
const keyToDelete = jwts.keys().next().value;
|
|
113804
|
+
if (keyToDelete) {
|
|
113805
|
+
jwts.delete(keyToDelete);
|
|
113806
|
+
}
|
|
113807
|
+
}
|
|
113808
|
+
jwts.set(key, jwt2);
|
|
113809
|
+
return {
|
|
113810
|
+
accessToken: json.accessToken,
|
|
113811
|
+
casUrl: json.casUrl
|
|
113812
|
+
};
|
|
113813
|
+
})();
|
|
113814
|
+
jwtPromises.set(key, promise);
|
|
113815
|
+
return promise;
|
|
113816
|
+
}
|
|
113817
|
+
|
|
113818
|
+
// src/utils/WebBlob.ts
|
|
113819
|
+
var WebBlob = class extends Blob {
|
|
113820
|
+
static async create(url, opts) {
|
|
113821
|
+
const customFetch = opts?.fetch ?? fetch;
|
|
113822
|
+
const response = await customFetch(url, {
|
|
113823
|
+
method: "HEAD",
|
|
113824
|
+
...opts?.accessToken && {
|
|
113825
|
+
headers: {
|
|
113826
|
+
Authorization: `Bearer ${opts.accessToken}`
|
|
113827
|
+
}
|
|
113828
|
+
}
|
|
113829
|
+
});
|
|
113830
|
+
const size = Number(response.headers.get("content-length"));
|
|
113831
|
+
const contentType = response.headers.get("content-type") || "";
|
|
113832
|
+
const supportRange = response.headers.get("accept-ranges") === "bytes";
|
|
113833
|
+
if (!supportRange || size < (opts?.cacheBelow ?? 1e6)) {
|
|
113834
|
+
return await (await customFetch(url)).blob();
|
|
113835
|
+
}
|
|
113836
|
+
return new WebBlob(url, 0, size, contentType, true, customFetch, opts?.accessToken);
|
|
113837
|
+
}
|
|
113838
|
+
url;
|
|
113839
|
+
start;
|
|
113840
|
+
end;
|
|
113841
|
+
contentType;
|
|
113842
|
+
full;
|
|
113843
|
+
fetch;
|
|
113844
|
+
accessToken;
|
|
113845
|
+
constructor(url, start, end, contentType, full, customFetch, accessToken) {
|
|
113846
|
+
super([]);
|
|
113847
|
+
this.url = url;
|
|
113848
|
+
this.start = start;
|
|
113849
|
+
this.end = end;
|
|
113850
|
+
this.contentType = contentType;
|
|
113851
|
+
this.full = full;
|
|
113852
|
+
this.fetch = customFetch;
|
|
113853
|
+
this.accessToken = accessToken;
|
|
113854
|
+
}
|
|
113855
|
+
get size() {
|
|
113856
|
+
return this.end - this.start;
|
|
113857
|
+
}
|
|
113858
|
+
get type() {
|
|
113859
|
+
return this.contentType;
|
|
113860
|
+
}
|
|
113861
|
+
slice(start = 0, end = this.size) {
|
|
113862
|
+
const slice = new WebBlob(
|
|
113863
|
+
this.url,
|
|
113864
|
+
this.start + start,
|
|
113865
|
+
Math.min(this.start + end, this.end),
|
|
113866
|
+
this.contentType,
|
|
113867
|
+
start === 0 && end === this.size ? this.full : false,
|
|
113868
|
+
this.fetch,
|
|
113869
|
+
this.accessToken
|
|
113870
|
+
);
|
|
113871
|
+
return slice;
|
|
113872
|
+
}
|
|
113873
|
+
async arrayBuffer() {
|
|
113874
|
+
const result = await this.fetchRange();
|
|
113875
|
+
return result.arrayBuffer();
|
|
113876
|
+
}
|
|
113877
|
+
async text() {
|
|
113878
|
+
const result = await this.fetchRange();
|
|
113879
|
+
return result.text();
|
|
113880
|
+
}
|
|
113881
|
+
stream() {
|
|
113882
|
+
const stream = new TransformStream();
|
|
113883
|
+
this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
113884
|
+
return stream.readable;
|
|
113885
|
+
}
|
|
113886
|
+
fetchRange() {
|
|
113887
|
+
const fetch2 = this.fetch;
|
|
113888
|
+
if (this.full) {
|
|
113889
|
+
return fetch2(this.url, {
|
|
113890
|
+
...this.accessToken && {
|
|
113891
|
+
headers: {
|
|
113892
|
+
Authorization: `Bearer ${this.accessToken}`
|
|
113893
|
+
}
|
|
113894
|
+
}
|
|
113895
|
+
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
113896
|
+
}
|
|
113897
|
+
return fetch2(this.url, {
|
|
113898
|
+
headers: {
|
|
113899
|
+
Range: `bytes=${this.start}-${this.end - 1}`,
|
|
113900
|
+
...this.accessToken && { Authorization: `Bearer ${this.accessToken}` }
|
|
113901
|
+
}
|
|
113902
|
+
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
113903
|
+
}
|
|
113904
|
+
};
|
|
113905
|
+
|
|
113906
|
+
// src/utils/parseLinkHeader.ts
|
|
113907
|
+
function parseLinkHeader(header) {
|
|
113908
|
+
const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g;
|
|
113909
|
+
return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url]));
|
|
113910
|
+
}
|
|
113911
|
+
|
|
113912
|
+
// src/lib/file-download-info.ts
|
|
113913
|
+
async function fileDownloadInfo(params) {
|
|
113914
|
+
const accessToken = checkCredentials(params);
|
|
113915
|
+
const repoId = toRepoId(params.repo);
|
|
113916
|
+
const hubUrl = params.hubUrl ?? HUB_URL;
|
|
113917
|
+
const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}/${encodeURIComponent(params.revision ?? "main")}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : "");
|
|
113918
|
+
const resp = await (params.fetch ?? fetch)(url, {
|
|
113919
|
+
method: "GET",
|
|
113920
|
+
headers: {
|
|
113921
|
+
...accessToken && {
|
|
113922
|
+
Authorization: `Bearer ${accessToken}`
|
|
113923
|
+
},
|
|
113924
|
+
Range: "bytes=0-0",
|
|
113925
|
+
Accept: "application/vnd.xet-fileinfo+json, */*"
|
|
113926
|
+
}
|
|
113927
|
+
});
|
|
113928
|
+
if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") {
|
|
113929
|
+
return null;
|
|
113930
|
+
}
|
|
113931
|
+
if (!resp.ok) {
|
|
113932
|
+
throw await createApiError(resp);
|
|
113933
|
+
}
|
|
113934
|
+
let size;
|
|
113935
|
+
let xetInfo;
|
|
113936
|
+
if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) {
|
|
113937
|
+
size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid");
|
|
113938
|
+
if (isNaN(size)) {
|
|
113939
|
+
throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header");
|
|
113940
|
+
}
|
|
113941
|
+
const hash2 = resp.headers.get("X-Xet-Hash");
|
|
113942
|
+
const links = parseLinkHeader(resp.headers.get("Link") ?? "");
|
|
113943
|
+
const reconstructionUrl = (() => {
|
|
113944
|
+
try {
|
|
113945
|
+
return new URL(links["xet-reconstruction-info"]);
|
|
113946
|
+
} catch {
|
|
113947
|
+
return null;
|
|
113948
|
+
}
|
|
113949
|
+
})();
|
|
113950
|
+
const refreshUrl = (() => {
|
|
113951
|
+
try {
|
|
113952
|
+
return new URL(links["xet-auth"]);
|
|
113953
|
+
} catch {
|
|
113954
|
+
return null;
|
|
113955
|
+
}
|
|
113956
|
+
})();
|
|
113957
|
+
if (!hash2) {
|
|
113958
|
+
throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header");
|
|
113959
|
+
}
|
|
113960
|
+
if (!reconstructionUrl || !refreshUrl) {
|
|
113961
|
+
throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header");
|
|
113962
|
+
}
|
|
113963
|
+
xetInfo = {
|
|
113964
|
+
hash: hash2,
|
|
113965
|
+
refreshUrl,
|
|
113966
|
+
reconstructionUrl
|
|
113967
|
+
};
|
|
113968
|
+
}
|
|
113969
|
+
if (size === void 0 || isNaN(size)) {
|
|
113970
|
+
const contentRangeHeader = resp.headers.get("content-range");
|
|
113971
|
+
if (!contentRangeHeader) {
|
|
113972
|
+
throw new InvalidApiResponseFormatError("Expected size information");
|
|
113973
|
+
}
|
|
113974
|
+
const [, parsedSize] = contentRangeHeader.split("/");
|
|
113975
|
+
size = parseInt(parsedSize);
|
|
113976
|
+
if (isNaN(size)) {
|
|
113977
|
+
throw new InvalidApiResponseFormatError("Invalid file size received");
|
|
113978
|
+
}
|
|
113979
|
+
}
|
|
113980
|
+
const etag = resp.headers.get("X-Linked-ETag") ?? resp.headers.get("ETag") ?? void 0;
|
|
113981
|
+
if (!etag) {
|
|
113982
|
+
throw new InvalidApiResponseFormatError("Expected ETag");
|
|
113983
|
+
}
|
|
113984
|
+
return {
|
|
113985
|
+
etag,
|
|
113986
|
+
size,
|
|
113987
|
+
xet: xetInfo,
|
|
113988
|
+
// Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it.
|
|
113989
|
+
url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url
|
|
113990
|
+
};
|
|
113991
|
+
}
|
|
113992
|
+
|
|
113993
|
+
// src/lib/download-file.ts
|
|
113994
|
+
async function downloadFile(params) {
|
|
113995
|
+
const accessToken = checkCredentials(params);
|
|
113996
|
+
const info = params.downloadInfo ?? await fileDownloadInfo({
|
|
113997
|
+
accessToken,
|
|
113998
|
+
repo: params.repo,
|
|
113999
|
+
path: params.path,
|
|
114000
|
+
revision: params.revision,
|
|
114001
|
+
hubUrl: params.hubUrl,
|
|
114002
|
+
fetch: params.fetch,
|
|
114003
|
+
raw: params.raw
|
|
114004
|
+
});
|
|
114005
|
+
if (!info) {
|
|
114006
|
+
return null;
|
|
114007
|
+
}
|
|
114008
|
+
if (info.xet && params.xet) {
|
|
114009
|
+
return new XetBlob({
|
|
114010
|
+
refreshUrl: info.xet.refreshUrl.href,
|
|
114011
|
+
reconstructionUrl: info.xet.reconstructionUrl.href,
|
|
114012
|
+
fetch: params.fetch,
|
|
114013
|
+
accessToken,
|
|
114014
|
+
size: info.size
|
|
114015
|
+
});
|
|
114016
|
+
}
|
|
114017
|
+
return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken);
|
|
114018
|
+
}
|
|
114019
|
+
|
|
114020
|
+
// src/lib/list-files.ts
|
|
114021
|
+
async function* listFiles(params) {
|
|
114022
|
+
const accessToken = checkCredentials(params);
|
|
114023
|
+
const repoId = toRepoId(params.repo);
|
|
114024
|
+
let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree/${params.revision || "main"}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`;
|
|
114025
|
+
while (url) {
|
|
114026
|
+
const res = await (params.fetch ?? fetch)(url, {
|
|
114027
|
+
headers: {
|
|
114028
|
+
accept: "application/json",
|
|
114029
|
+
...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0
|
|
114030
|
+
}
|
|
114031
|
+
});
|
|
114032
|
+
if (!res.ok) {
|
|
114033
|
+
throw await createApiError(res);
|
|
114034
|
+
}
|
|
114035
|
+
const items = await res.json();
|
|
114036
|
+
for (const item of items) {
|
|
114037
|
+
yield item;
|
|
114038
|
+
}
|
|
114039
|
+
const linkHeader = res.headers.get("Link");
|
|
114040
|
+
url = linkHeader ? parseLinkHeader(linkHeader).next : void 0;
|
|
114041
|
+
}
|
|
114042
|
+
}
|
|
114043
|
+
|
|
114044
|
+
const DIR_BASED_ENGINES = new Set(["exllamav3", "mlx-lm"]);
|
|
114045
|
+
const FLAG_BASED_ENGINE_ARGS = {
|
|
114046
|
+
"llama.cpp": "--chat-template-file",
|
|
114047
|
+
sglang: "--chat-template",
|
|
114048
|
+
vllm: "--chat-template"
|
|
114049
|
+
};
|
|
114050
|
+
function getChatTemplateLocalPath(targetDirectory) {
|
|
114051
|
+
return join(targetDirectory, CHAT_TEMPLATE_LOCAL_FILE_NAME);
|
|
114052
|
+
}
|
|
114053
|
+
async function fetchTemplateContent({ huggingFaceToken, override }) {
|
|
114054
|
+
const accessToken = huggingFaceToken ?? process.env.HF_TOKEN ?? undefined;
|
|
114055
|
+
const filePath = override.filePath || CHAT_TEMPLATE_DEFAULT_FILE_PATH;
|
|
114056
|
+
const blob = await downloadFile({
|
|
114057
|
+
accessToken,
|
|
114058
|
+
path: filePath,
|
|
114059
|
+
repo: override.repo
|
|
114060
|
+
});
|
|
114061
|
+
if (!blob) {
|
|
114062
|
+
throw new Error(`Chat template file not found: ${override.repo}/${filePath}`);
|
|
114063
|
+
}
|
|
114064
|
+
const content = await blob.text();
|
|
114065
|
+
if (!content.trim()) {
|
|
114066
|
+
throw new Error(`Chat template file is empty: ${override.repo}/${filePath}`);
|
|
114067
|
+
}
|
|
114068
|
+
return content;
|
|
114069
|
+
}
|
|
114070
|
+
/**
|
|
114071
|
+
* Materializes the model's chat template override into the model directory:
|
|
114072
|
+
* - Always writes the canonical copy for flag-based engines.
|
|
114073
|
+
* - For engines that read the model directory (exllamav3, mlx-lm), also writes
|
|
114074
|
+
* the transformers-style `chat_template.jinja` drop-in.
|
|
114075
|
+
* - When no override is configured, removes any previously materialized files,
|
|
114076
|
+
* restoring the model repo's own `chat_template.jinja` if we replaced it.
|
|
114077
|
+
*/
|
|
114078
|
+
async function materializeChatTemplate({ engine, huggingFaceToken, model, targetDirectory }) {
|
|
114079
|
+
const override = model.chatTemplate ?? null;
|
|
114080
|
+
const canonicalPath = getChatTemplateLocalPath(targetDirectory);
|
|
114081
|
+
const dropInPath = join(targetDirectory, CHAT_TEMPLATE_DEFAULT_FILE_PATH);
|
|
114082
|
+
if (!override) {
|
|
114083
|
+
const markerPath = `${canonicalPath}.override`;
|
|
114084
|
+
if (existsSync(markerPath)) {
|
|
114085
|
+
await rm(dropInPath, { force: true });
|
|
114086
|
+
await rm(markerPath, { force: true });
|
|
114087
|
+
if (model.source.type === "huggingface") {
|
|
114088
|
+
await restoreOriginalTemplate({
|
|
114089
|
+
huggingFaceToken,
|
|
114090
|
+
modelSlug: model.source.slug,
|
|
114091
|
+
targetDirectory
|
|
114092
|
+
});
|
|
114093
|
+
}
|
|
114094
|
+
}
|
|
114095
|
+
await rm(canonicalPath, { force: true });
|
|
114096
|
+
return;
|
|
114097
|
+
}
|
|
114098
|
+
const content = override.type === "inline"
|
|
114099
|
+
? override.content
|
|
114100
|
+
: await fetchTemplateContent({ huggingFaceToken, override });
|
|
114101
|
+
await mkdir(targetDirectory, { recursive: true });
|
|
114102
|
+
await writeFile(canonicalPath, content, "utf8");
|
|
114103
|
+
if (DIR_BASED_ENGINES.has(engine)) {
|
|
114104
|
+
await writeFile(dropInPath, content, "utf8");
|
|
114105
|
+
await writeFile(`${canonicalPath}.override`, "1", "utf8");
|
|
114106
|
+
}
|
|
114107
|
+
}
|
|
114108
|
+
async function restoreOriginalTemplate({ huggingFaceToken, modelSlug, targetDirectory }) {
|
|
114109
|
+
const accessToken = huggingFaceToken ?? process.env.HF_TOKEN ?? undefined;
|
|
114110
|
+
try {
|
|
114111
|
+
const blob = await downloadFile({
|
|
114112
|
+
accessToken,
|
|
114113
|
+
path: CHAT_TEMPLATE_DEFAULT_FILE_PATH,
|
|
114114
|
+
repo: modelSlug.split(":")[0].split("@")[0]
|
|
114115
|
+
});
|
|
114116
|
+
if (blob) {
|
|
114117
|
+
const content = await blob.text();
|
|
114118
|
+
if (content.trim()) {
|
|
114119
|
+
await writeFile(join(targetDirectory, CHAT_TEMPLATE_DEFAULT_FILE_PATH), content, "utf8");
|
|
114120
|
+
}
|
|
114121
|
+
}
|
|
114122
|
+
}
|
|
114123
|
+
catch (error) {
|
|
114124
|
+
console.warn("[chatTemplate] Failed to restore original chat_template.jinja after override removal:", asError(error).message);
|
|
114125
|
+
}
|
|
114126
|
+
}
|
|
114127
|
+
/**
|
|
114128
|
+
* CLI arguments applying the materialized chat template for flag-based engines.
|
|
114129
|
+
* Returns an empty array when no override exists, the engine applies templates
|
|
114130
|
+
* from the model directory, or the engine does not support overrides.
|
|
114131
|
+
*/
|
|
114132
|
+
async function getChatTemplateEngineArgs({ engine, model, targetDirectory }) {
|
|
114133
|
+
if (!model.chatTemplate)
|
|
114134
|
+
return [];
|
|
114135
|
+
const flag = FLAG_BASED_ENGINE_ARGS[engine];
|
|
114136
|
+
if (!flag) {
|
|
114137
|
+
if (engine === "tensorrt-llm") {
|
|
114138
|
+
console.warn("[chatTemplate] TensorRT-LLM does not support chat template overrides; ignoring");
|
|
114139
|
+
}
|
|
114140
|
+
return [];
|
|
114141
|
+
}
|
|
114142
|
+
const templatePath = getChatTemplateLocalPath(targetDirectory);
|
|
114143
|
+
if (!existsSync(templatePath)) {
|
|
114144
|
+
console.warn(`[chatTemplate] Template file missing for ${engine}; continuing with embedded template`);
|
|
114145
|
+
return [];
|
|
114146
|
+
}
|
|
114147
|
+
return [flag, templatePath];
|
|
114148
|
+
}
|
|
114149
|
+
/**
|
|
114150
|
+
* Post-start verification for llama.cpp: compares the template the server
|
|
114151
|
+
* reports via `/props` against the materialized override file. Returns a
|
|
114152
|
+
* warning message on mismatch, or null when the template applied cleanly
|
|
114153
|
+
* (or verification is not applicable).
|
|
114154
|
+
*/
|
|
114155
|
+
async function verifyEngineChatTemplate({ engine, enginePort, logger, model, targetDirectory }) {
|
|
114156
|
+
if (!model.chatTemplate || engine !== "llama.cpp")
|
|
114157
|
+
return null;
|
|
114158
|
+
try {
|
|
114159
|
+
const templatePath = getChatTemplateLocalPath(targetDirectory);
|
|
114160
|
+
if (!existsSync(templatePath)) {
|
|
114161
|
+
return "Chat template override file missing on disk";
|
|
114162
|
+
}
|
|
114163
|
+
const [propsResponse, expected] = await Promise.all([
|
|
114164
|
+
fetch(`http://localhost:${enginePort}/props`, {
|
|
114165
|
+
signal: AbortSignal.timeout(5000)
|
|
114166
|
+
}),
|
|
114167
|
+
readFile(templatePath, "utf8")
|
|
114168
|
+
]);
|
|
114169
|
+
if (!propsResponse.ok) {
|
|
114170
|
+
return `Chat template verification unavailable: /props returned ${propsResponse.status}`;
|
|
114171
|
+
}
|
|
114172
|
+
const props = (await propsResponse.json());
|
|
114173
|
+
const served = typeof props.chat_template === "string" ? props.chat_template.trim() : "";
|
|
114174
|
+
if (served !== expected.trim()) {
|
|
114175
|
+
return "Chat template override did not apply: served template differs from configured template";
|
|
114176
|
+
}
|
|
114177
|
+
return null;
|
|
114178
|
+
}
|
|
114179
|
+
catch (error) {
|
|
114180
|
+
logger.warn("Chat template verification failed", {
|
|
114181
|
+
error: asError(error)
|
|
114182
|
+
});
|
|
114183
|
+
return null;
|
|
114184
|
+
}
|
|
114185
|
+
}
|
|
114186
|
+
|
|
114187
|
+
const SECRET_FLAGS = new Set([
|
|
114188
|
+
"api-key",
|
|
114189
|
+
"auth-token",
|
|
114190
|
+
"hf-token",
|
|
114191
|
+
"key",
|
|
114192
|
+
"password",
|
|
114193
|
+
"secret",
|
|
114194
|
+
"token"
|
|
114195
|
+
]);
|
|
114196
|
+
function redactSecretArgs(args) {
|
|
114197
|
+
return args.map((arg, index) => {
|
|
114198
|
+
const equalsIndex = arg.indexOf("=");
|
|
114199
|
+
if (arg.startsWith("--") && equalsIndex > 0) {
|
|
114200
|
+
const flag = arg.slice(2, equalsIndex);
|
|
114201
|
+
if (SECRET_FLAGS.has(flag)) {
|
|
114202
|
+
return `${arg.slice(0, equalsIndex + 1)}***`;
|
|
114203
|
+
}
|
|
114204
|
+
}
|
|
114205
|
+
const previous = args[index - 1];
|
|
114206
|
+
if (previous &&
|
|
114207
|
+
previous.startsWith("--") &&
|
|
114208
|
+
!previous.includes("=") &&
|
|
114209
|
+
SECRET_FLAGS.has(previous.slice(2))) {
|
|
114210
|
+
return "***";
|
|
114211
|
+
}
|
|
114212
|
+
return arg;
|
|
114213
|
+
});
|
|
114214
|
+
}
|
|
114215
|
+
async function createEngineProcess({ args, bin, logger }) {
|
|
114216
|
+
logger.info("Starting engine process", {
|
|
114217
|
+
command: { args: redactSecretArgs(args), bin }
|
|
114218
|
+
});
|
|
114219
|
+
const processManager = new ProcessManager({ args, command: bin });
|
|
114220
|
+
await processManager.start();
|
|
114221
|
+
return processManager;
|
|
114222
|
+
}
|
|
114223
|
+
|
|
114224
|
+
function parseExtraArgs(extraArgs) {
|
|
114225
|
+
if (!Array.isArray(extraArgs) ||
|
|
114226
|
+
!extraArgs.every((value) => typeof value === "string")) {
|
|
114227
|
+
return [];
|
|
114228
|
+
}
|
|
114229
|
+
return extraArgs.flatMap(tokenizeShellLine);
|
|
114230
|
+
}
|
|
114231
|
+
function tokenizeShellLine(input) {
|
|
114232
|
+
const tokens = [];
|
|
114233
|
+
let buffer = "";
|
|
114234
|
+
let inQuote = null;
|
|
114235
|
+
let hasBuffer = false;
|
|
114236
|
+
for (let index = 0; index < input.length; index += 1) {
|
|
114237
|
+
const char = input[index];
|
|
114238
|
+
if (inQuote) {
|
|
114239
|
+
if (char === inQuote) {
|
|
114240
|
+
inQuote = null;
|
|
114241
|
+
}
|
|
114242
|
+
else {
|
|
114243
|
+
buffer += char;
|
|
114244
|
+
}
|
|
114245
|
+
hasBuffer = true;
|
|
114246
|
+
}
|
|
114247
|
+
else if (char === '"' || char === "'") {
|
|
114248
|
+
inQuote = char;
|
|
114249
|
+
hasBuffer = true;
|
|
114250
|
+
}
|
|
114251
|
+
else if (char === " " || char === "\t") {
|
|
114252
|
+
if (hasBuffer) {
|
|
114253
|
+
tokens.push(buffer);
|
|
114254
|
+
buffer = "";
|
|
114255
|
+
hasBuffer = false;
|
|
114256
|
+
}
|
|
114257
|
+
}
|
|
114258
|
+
else {
|
|
114259
|
+
buffer += char;
|
|
114260
|
+
hasBuffer = true;
|
|
114261
|
+
}
|
|
114262
|
+
}
|
|
114263
|
+
if (hasBuffer) {
|
|
114264
|
+
tokens.push(buffer);
|
|
114265
|
+
}
|
|
114266
|
+
return tokens;
|
|
114267
|
+
}
|
|
114268
|
+
|
|
114269
|
+
const balanced = (a, b, str) => {
|
|
114270
|
+
const ma = a instanceof RegExp ? maybeMatch(a, str) : a;
|
|
114271
|
+
const mb = b instanceof RegExp ? maybeMatch(b, str) : b;
|
|
114272
|
+
const r = ma !== null && mb != null && range(ma, mb, str);
|
|
114273
|
+
return (r && {
|
|
114274
|
+
start: r[0],
|
|
114275
|
+
end: r[1],
|
|
114276
|
+
pre: str.slice(0, r[0]),
|
|
114277
|
+
body: str.slice(r[0] + ma.length, r[1]),
|
|
114278
|
+
post: str.slice(r[1] + mb.length),
|
|
114279
|
+
});
|
|
114280
|
+
};
|
|
114281
|
+
const maybeMatch = (reg, str) => {
|
|
114282
|
+
const m = str.match(reg);
|
|
114283
|
+
return m ? m[0] : null;
|
|
114284
|
+
};
|
|
114285
|
+
const range = (a, b, str) => {
|
|
114286
|
+
let begs, beg, left, right = undefined, result;
|
|
114287
|
+
let ai = str.indexOf(a);
|
|
114288
|
+
let bi = str.indexOf(b, ai + 1);
|
|
114289
|
+
let i = ai;
|
|
114290
|
+
if (ai >= 0 && bi > 0) {
|
|
114291
|
+
if (a === b) {
|
|
114292
|
+
return [ai, bi];
|
|
114293
|
+
}
|
|
114294
|
+
begs = [];
|
|
114295
|
+
left = str.length;
|
|
114296
|
+
while (i >= 0 && !result) {
|
|
114297
|
+
if (i === ai) {
|
|
114298
|
+
begs.push(i);
|
|
114299
|
+
ai = str.indexOf(a, i + 1);
|
|
114300
|
+
}
|
|
114301
|
+
else if (begs.length === 1) {
|
|
114302
|
+
const r = begs.pop();
|
|
114303
|
+
if (r !== undefined)
|
|
114304
|
+
result = [r, bi];
|
|
114305
|
+
}
|
|
114306
|
+
else {
|
|
114307
|
+
beg = begs.pop();
|
|
114308
|
+
if (beg !== undefined && beg < left) {
|
|
114309
|
+
left = beg;
|
|
114310
|
+
right = bi;
|
|
114311
|
+
}
|
|
114312
|
+
bi = str.indexOf(b, i + 1);
|
|
114313
|
+
}
|
|
114314
|
+
i = ai < bi && ai >= 0 ? ai : bi;
|
|
114315
|
+
}
|
|
114316
|
+
if (begs.length && right !== undefined) {
|
|
114317
|
+
result = [left, right];
|
|
114318
|
+
}
|
|
114319
|
+
}
|
|
114320
|
+
return result;
|
|
114321
|
+
};
|
|
114322
|
+
|
|
114323
|
+
const escSlash = '\0SLASH' + Math.random() + '\0';
|
|
114324
|
+
const escOpen = '\0OPEN' + Math.random() + '\0';
|
|
114325
|
+
const escClose = '\0CLOSE' + Math.random() + '\0';
|
|
114326
|
+
const escComma = '\0COMMA' + Math.random() + '\0';
|
|
114327
|
+
const escPeriod = '\0PERIOD' + Math.random() + '\0';
|
|
114328
|
+
const escSlashPattern = new RegExp(escSlash, 'g');
|
|
114329
|
+
const escOpenPattern = new RegExp(escOpen, 'g');
|
|
114330
|
+
const escClosePattern = new RegExp(escClose, 'g');
|
|
114331
|
+
const escCommaPattern = new RegExp(escComma, 'g');
|
|
114332
|
+
const escPeriodPattern = new RegExp(escPeriod, 'g');
|
|
114333
|
+
const slashPattern = /\\\\/g;
|
|
114334
|
+
const openPattern = /\\{/g;
|
|
114335
|
+
const closePattern = /\\}/g;
|
|
114336
|
+
const commaPattern = /\\,/g;
|
|
114337
|
+
const periodPattern = /\\\./g;
|
|
114338
|
+
const EXPANSION_MAX = 100_000;
|
|
114339
|
+
// `EXPANSION_MAX` caps the *number* of expansions, but not their length. An
|
|
114340
|
+
// input like `'{a,b}'.repeat(1500)` stays under that count - its output is
|
|
114341
|
+
// truncated to 100k results - while making every result ~1500 characters
|
|
114342
|
+
// long. The result set, and the intermediate arrays built while combining
|
|
114343
|
+
// brace sets, then grow large enough to exhaust memory and crash the process
|
|
114344
|
+
// (CVE-2026-14257). `EXPANSION_MAX_LENGTH` bounds the total number of
|
|
114345
|
+
// characters the accumulator may hold at any point, so memory stays flat no
|
|
114346
|
+
// matter how many brace groups are chained. The limit sits well above any
|
|
114347
|
+
// realistic expansion (100k results hitting `EXPANSION_MAX` measure ~1M
|
|
114348
|
+
// characters) so legitimate input is unaffected.
|
|
114349
|
+
const EXPANSION_MAX_LENGTH = 4_000_000;
|
|
114350
|
+
function numeric(str) {
|
|
114351
|
+
return !isNaN(str) ? parseInt(str, 10) : str.charCodeAt(0);
|
|
114352
|
+
}
|
|
114353
|
+
function escapeBraces(str) {
|
|
114354
|
+
return str
|
|
114355
|
+
.replace(slashPattern, escSlash)
|
|
114356
|
+
.replace(openPattern, escOpen)
|
|
114357
|
+
.replace(closePattern, escClose)
|
|
114358
|
+
.replace(commaPattern, escComma)
|
|
114359
|
+
.replace(periodPattern, escPeriod);
|
|
114360
|
+
}
|
|
114361
|
+
function unescapeBraces(str) {
|
|
114362
|
+
return str
|
|
114363
|
+
.replace(escSlashPattern, '\\')
|
|
114364
|
+
.replace(escOpenPattern, '{')
|
|
114365
|
+
.replace(escClosePattern, '}')
|
|
114366
|
+
.replace(escCommaPattern, ',')
|
|
114367
|
+
.replace(escPeriodPattern, '.');
|
|
114368
|
+
}
|
|
114369
|
+
/**
|
|
114370
|
+
* Basically just str.split(","), but handling cases
|
|
114371
|
+
* where we have nested braced sections, which should be
|
|
114372
|
+
* treated as individual members, like {a,{b,c},d}
|
|
114373
|
+
*/
|
|
114374
|
+
function parseCommaParts(str) {
|
|
114375
|
+
if (!str) {
|
|
114376
|
+
return [''];
|
|
114377
|
+
}
|
|
114378
|
+
const parts = [];
|
|
114379
|
+
const m = balanced('{', '}', str);
|
|
114380
|
+
if (!m) {
|
|
114381
|
+
return str.split(',');
|
|
114382
|
+
}
|
|
114383
|
+
const { pre, body, post } = m;
|
|
114384
|
+
const p = pre.split(',');
|
|
114385
|
+
p[p.length - 1] += '{' + body + '}';
|
|
114386
|
+
const postParts = parseCommaParts(post);
|
|
114387
|
+
if (post.length) {
|
|
114388
|
+
p[p.length - 1] += postParts.shift();
|
|
114389
|
+
p.push.apply(p, postParts);
|
|
114390
|
+
}
|
|
114391
|
+
parts.push.apply(parts, p);
|
|
114392
|
+
return parts;
|
|
114393
|
+
}
|
|
114394
|
+
function expand(str, options = {}) {
|
|
114395
|
+
if (!str) {
|
|
114396
|
+
return [];
|
|
114397
|
+
}
|
|
114398
|
+
const { max = EXPANSION_MAX, maxLength = EXPANSION_MAX_LENGTH } = options;
|
|
114399
|
+
// I don't know why Bash 4.3 does this, but it does.
|
|
114400
|
+
// Anything starting with {} will have the first two bytes preserved
|
|
114401
|
+
// but *only* at the top level, so {},a}b will not expand to anything,
|
|
114402
|
+
// but a{},b}c will be expanded to [a}c,abc].
|
|
114403
|
+
// One could argue that this is a bug in Bash, but since the goal of
|
|
114404
|
+
// this module is to match Bash's rules, we escape a leading {}
|
|
114405
|
+
if (str.slice(0, 2) === '{}') {
|
|
114406
|
+
str = '\\{\\}' + str.slice(2);
|
|
114407
|
+
}
|
|
114408
|
+
return expand_(escapeBraces(str), max, maxLength, true).map(unescapeBraces);
|
|
114409
|
+
}
|
|
114410
|
+
function embrace(str) {
|
|
114411
|
+
return '{' + str + '}';
|
|
114412
|
+
}
|
|
114413
|
+
function isPadded(el) {
|
|
114414
|
+
return /^-?0\d/.test(el);
|
|
114415
|
+
}
|
|
114416
|
+
function lte(i, y) {
|
|
114417
|
+
return i <= y;
|
|
114418
|
+
}
|
|
114419
|
+
function gte(i, y) {
|
|
114420
|
+
return i >= y;
|
|
114421
|
+
}
|
|
114422
|
+
// Build `{ acc[a] + pre + values[v] }` for every combination, capping the
|
|
114423
|
+
// number of results at `max` and the total number of characters at `maxLength`.
|
|
114424
|
+
// This is the one place output grows, so bounding it here keeps the single
|
|
114425
|
+
// accumulator - and therefore memory - flat regardless of how many brace groups
|
|
114426
|
+
// are combined (CVE-2026-14257).
|
|
114427
|
+
function combine(acc, pre, values, max, maxLength, dropEmpties) {
|
|
114428
|
+
const out = [];
|
|
114429
|
+
let length = 0;
|
|
114430
|
+
for (let a = 0; a < acc.length; a++) {
|
|
114431
|
+
for (let v = 0; v < values.length; v++) {
|
|
114432
|
+
if (out.length >= max)
|
|
114433
|
+
return out;
|
|
114434
|
+
const expansion = acc[a] + pre + values[v];
|
|
114435
|
+
// Bash drops empty results at the top level. Skip them before they count
|
|
114436
|
+
// against `max`, so `max` bounds the number of *kept* results.
|
|
114437
|
+
if (dropEmpties && !expansion)
|
|
114438
|
+
continue;
|
|
114439
|
+
if (length + expansion.length > maxLength)
|
|
114440
|
+
return out;
|
|
114441
|
+
out.push(expansion);
|
|
114442
|
+
length += expansion.length;
|
|
114443
|
+
}
|
|
114444
|
+
}
|
|
114445
|
+
return out;
|
|
114446
|
+
}
|
|
114447
|
+
// The expansion values of a single numeric (`1..5`) or alphabetic (`a..e..2`)
|
|
114448
|
+
// sequence body.
|
|
114449
|
+
function expandSequence(body, isAlphaSequence, max, maxLength) {
|
|
114450
|
+
const n = body.split(/\.\./);
|
|
114451
|
+
const N = [];
|
|
114452
|
+
// A sequence body always splits into two or three parts, but the compiler
|
|
114453
|
+
// can't know that.
|
|
114454
|
+
/* c8 ignore start */
|
|
114455
|
+
if (n[0] === undefined || n[1] === undefined) {
|
|
114456
|
+
return N;
|
|
114457
|
+
}
|
|
114458
|
+
/* c8 ignore stop */
|
|
114459
|
+
const x = numeric(n[0]);
|
|
114460
|
+
const y = numeric(n[1]);
|
|
114461
|
+
const width = Math.max(n[0].length, n[1].length);
|
|
114462
|
+
let incr = n.length === 3 && n[2] !== undefined ?
|
|
114463
|
+
Math.max(Math.abs(numeric(n[2])), 1)
|
|
114464
|
+
: 1;
|
|
114465
|
+
let test = lte;
|
|
114466
|
+
const reverse = y < x;
|
|
114467
|
+
if (reverse) {
|
|
114468
|
+
incr *= -1;
|
|
114469
|
+
test = gte;
|
|
114470
|
+
}
|
|
114471
|
+
const pad = n.some(isPadded);
|
|
114472
|
+
let length = 0;
|
|
114473
|
+
for (let i = x; test(i, y) && N.length < max; i += incr) {
|
|
114474
|
+
let c;
|
|
114475
|
+
if (isAlphaSequence) {
|
|
114476
|
+
c = String.fromCharCode(i);
|
|
114477
|
+
if (c === '\\') {
|
|
114478
|
+
c = '';
|
|
114479
|
+
}
|
|
114480
|
+
}
|
|
114481
|
+
else {
|
|
114482
|
+
c = String(i);
|
|
114483
|
+
if (pad) {
|
|
114484
|
+
const need = width - c.length;
|
|
114485
|
+
if (need > 0) {
|
|
114486
|
+
const z = new Array(need + 1).join('0');
|
|
114487
|
+
if (i < 0) {
|
|
114488
|
+
c = '-' + z + c.slice(1);
|
|
114489
|
+
}
|
|
114490
|
+
else {
|
|
114491
|
+
c = z + c;
|
|
114492
|
+
}
|
|
114493
|
+
}
|
|
114494
|
+
}
|
|
114495
|
+
}
|
|
114496
|
+
if (length + c.length > maxLength)
|
|
114497
|
+
break;
|
|
114498
|
+
N.push(c);
|
|
114499
|
+
length += c.length;
|
|
114500
|
+
}
|
|
114501
|
+
return N;
|
|
114502
|
+
}
|
|
114503
|
+
function expand_(str, max, maxLength, isTop) {
|
|
114504
|
+
// Consume the string's top-level brace groups left to right, threading a
|
|
114505
|
+
// running set of combined prefixes (`acc`). Expanding the tail iteratively -
|
|
114506
|
+
// rather than recursing on `m.post` once per group - keeps the native stack
|
|
114507
|
+
// depth constant, so deeply chained input (`'{a,b}'.repeat(3000)`) can no
|
|
114508
|
+
// longer overflow the stack, and leaves a single accumulator whose size
|
|
114509
|
+
// `maxLength` bounds directly (CVE-2026-14257).
|
|
114510
|
+
let acc = [''];
|
|
114511
|
+
// Bash drops empty results, but only when the *first* top-level group is a
|
|
114512
|
+
// comma set - a sequence like `{a..\}` may legitimately yield ''. The drop
|
|
114513
|
+
// is on the final strings, so it is applied to whichever `combine` produces
|
|
114514
|
+
// them (the one with no brace set left in the tail).
|
|
114515
|
+
let dropEmpties = false;
|
|
114516
|
+
let firstGroup = true;
|
|
114517
|
+
for (;;) {
|
|
114518
|
+
const m = balanced('{', '}', str);
|
|
114519
|
+
// No brace set left: the rest of the string is literal.
|
|
114520
|
+
if (!m) {
|
|
114521
|
+
return combine(acc, str, [''], max, maxLength, dropEmpties);
|
|
114522
|
+
}
|
|
114523
|
+
// no need to expand pre, since it is guaranteed to be free of brace-sets
|
|
114524
|
+
const pre = m.pre;
|
|
114525
|
+
if (/\$$/.test(pre)) {
|
|
114526
|
+
acc = combine(acc, pre + '{' + m.body + '}', [''], max, maxLength, dropEmpties && !m.post.length);
|
|
114527
|
+
firstGroup = false;
|
|
114528
|
+
if (!m.post.length)
|
|
114529
|
+
break;
|
|
114530
|
+
str = m.post;
|
|
114531
|
+
continue;
|
|
114532
|
+
}
|
|
114533
|
+
const isNumericSequence = /^-?\d+\.\.-?\d+(?:\.\.-?\d+)?$/.test(m.body);
|
|
114534
|
+
const isAlphaSequence = /^[a-zA-Z]\.\.[a-zA-Z](?:\.\.-?\d+)?$/.test(m.body);
|
|
114535
|
+
const isSequence = isNumericSequence || isAlphaSequence;
|
|
114536
|
+
const isOptions = m.body.indexOf(',') >= 0;
|
|
114537
|
+
if (!isSequence && !isOptions) {
|
|
114538
|
+
// {a},b}
|
|
114539
|
+
if (m.post.match(/,(?!,).*\}/)) {
|
|
114540
|
+
str = m.pre + '{' + m.body + escClose + m.post;
|
|
114541
|
+
isTop = true;
|
|
114542
|
+
continue;
|
|
114543
|
+
}
|
|
114544
|
+
// Nothing here expands, so the whole remaining string is literal.
|
|
114545
|
+
return combine(acc, pre + '{' + m.body + '}' + m.post, [''], max, maxLength, dropEmpties);
|
|
114546
|
+
}
|
|
114547
|
+
if (firstGroup) {
|
|
114548
|
+
dropEmpties = isTop && !isSequence;
|
|
114549
|
+
firstGroup = false;
|
|
114550
|
+
}
|
|
114551
|
+
let values;
|
|
114552
|
+
if (isSequence) {
|
|
114553
|
+
values = expandSequence(m.body, isAlphaSequence, max, maxLength);
|
|
114554
|
+
}
|
|
114555
|
+
else {
|
|
114556
|
+
let n = parseCommaParts(m.body);
|
|
114557
|
+
if (n.length === 1 && n[0] !== undefined) {
|
|
114558
|
+
// x{{a,b}}y ==> x{a}y x{b}y
|
|
114559
|
+
n = expand_(n[0], max, maxLength, false).map(embrace);
|
|
114560
|
+
//XXX is this necessary? Can't seem to hit it in tests.
|
|
114561
|
+
/* c8 ignore start */
|
|
114562
|
+
if (n.length === 1) {
|
|
114563
|
+
acc = combine(acc, pre + n[0], [''], max, maxLength, dropEmpties && !m.post.length);
|
|
114564
|
+
if (!m.post.length)
|
|
114565
|
+
break;
|
|
114566
|
+
str = m.post;
|
|
114567
|
+
continue;
|
|
114568
|
+
}
|
|
114569
|
+
/* c8 ignore stop */
|
|
114570
|
+
}
|
|
114571
|
+
// Values that `combine` is going to drop as empty produce no result, so
|
|
114572
|
+
// they must not count against `max` - otherwise `{a,,b}` with `max: 2`
|
|
114573
|
+
// would stop at `['a', '']` and yield one result instead of two. Skipping
|
|
114574
|
+
// them outright keeps `values` bounded while leaving `max` a bound on
|
|
114575
|
+
// *kept* results.
|
|
114576
|
+
let dropsEmpties = dropEmpties && !m.post.length && !pre;
|
|
114577
|
+
for (let d = 0; dropsEmpties && d < acc.length; d++) {
|
|
114578
|
+
if (acc[d]) {
|
|
114579
|
+
dropsEmpties = false;
|
|
114580
|
+
}
|
|
114581
|
+
}
|
|
114582
|
+
values = [];
|
|
114583
|
+
let valuesLength = 0;
|
|
114584
|
+
outer: for (let j = 0; j < n.length; j++) {
|
|
114585
|
+
const expanded = expand_(n[j], max, maxLength, false);
|
|
114586
|
+
for (let k = 0; k < expanded.length; k++) {
|
|
114587
|
+
const v = expanded[k];
|
|
114588
|
+
if (dropsEmpties && !v)
|
|
114589
|
+
continue;
|
|
114590
|
+
if (values.length >= max || valuesLength + v.length > maxLength) {
|
|
114591
|
+
break outer;
|
|
114592
|
+
}
|
|
114593
|
+
values.push(v);
|
|
114594
|
+
valuesLength += v.length;
|
|
114595
|
+
}
|
|
114596
|
+
}
|
|
114597
|
+
}
|
|
114598
|
+
acc = combine(acc, pre, values, max, maxLength, dropEmpties && !m.post.length);
|
|
114599
|
+
if (!m.post.length)
|
|
114600
|
+
break;
|
|
114601
|
+
str = m.post;
|
|
114602
|
+
}
|
|
114603
|
+
return acc;
|
|
114604
|
+
}
|
|
114605
|
+
|
|
114606
|
+
const MAX_PATTERN_LENGTH = 1024 * 64;
|
|
114607
|
+
const assertValidPattern = (pattern) => {
|
|
114608
|
+
if (typeof pattern !== 'string') {
|
|
114609
|
+
throw new TypeError('invalid pattern');
|
|
114610
|
+
}
|
|
114611
|
+
if (pattern.length > MAX_PATTERN_LENGTH) {
|
|
114612
|
+
throw new TypeError('pattern is too long');
|
|
114613
|
+
}
|
|
114614
|
+
};
|
|
114615
|
+
|
|
114616
|
+
// translate the various posix character classes into unicode properties
|
|
114617
|
+
// this works across all unicode locales
|
|
114618
|
+
// { <posix class>: [<translation>, /u flag required, negated]
|
|
114619
|
+
const posixClasses = {
|
|
114620
|
+
'[:alnum:]': ['\\p{L}\\p{Nl}\\p{Nd}', true],
|
|
114621
|
+
'[:alpha:]': ['\\p{L}\\p{Nl}', true],
|
|
114622
|
+
'[:ascii:]': ['\\x' + '00-\\x' + '7f', false],
|
|
114623
|
+
'[:blank:]': ['\\p{Zs}\\t', true],
|
|
114624
|
+
'[:cntrl:]': ['\\p{Cc}', true],
|
|
114625
|
+
'[:digit:]': ['\\p{Nd}', true],
|
|
114626
|
+
'[:graph:]': ['\\p{Z}\\p{C}', true, true],
|
|
114627
|
+
'[:lower:]': ['\\p{Ll}', true],
|
|
114628
|
+
'[:print:]': ['\\p{C}', true],
|
|
114629
|
+
'[:punct:]': ['\\p{P}', true],
|
|
114630
|
+
'[:space:]': ['\\p{Z}\\t\\r\\n\\v\\f', true],
|
|
114631
|
+
'[:upper:]': ['\\p{Lu}', true],
|
|
114632
|
+
'[:word:]': ['\\p{L}\\p{Nl}\\p{Nd}\\p{Pc}', true],
|
|
114633
|
+
'[:xdigit:]': ['A-Fa-f0-9', false],
|
|
114634
|
+
};
|
|
114635
|
+
// only need to escape a few things inside of brace expressions
|
|
114636
|
+
// escapes: [ \ ] -
|
|
114637
|
+
const braceEscape = (s) => s.replace(/[[\]\\-]/g, '\\$&');
|
|
114638
|
+
// escape all regexp magic characters
|
|
114639
|
+
const regexpEscape = (s) => s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&');
|
|
114640
|
+
// everything has already been escaped, we just have to join
|
|
114641
|
+
const rangesToString = (ranges) => ranges.join('');
|
|
114642
|
+
// takes a glob string at a posix brace expression, and returns
|
|
114643
|
+
// an equivalent regular expression source, and boolean indicating
|
|
114644
|
+
// whether the /u flag needs to be applied, and the number of chars
|
|
114645
|
+
// consumed to parse the character class.
|
|
114646
|
+
// This also removes out of order ranges, and returns ($.) if the
|
|
114647
|
+
// entire class just no good.
|
|
114648
|
+
const parseClass = (glob, position) => {
|
|
114649
|
+
const pos = position;
|
|
114650
|
+
/* c8 ignore start */
|
|
114651
|
+
if (glob.charAt(pos) !== '[') {
|
|
114652
|
+
throw new Error('not in a brace expression');
|
|
114653
|
+
}
|
|
114654
|
+
/* c8 ignore stop */
|
|
114655
|
+
const ranges = [];
|
|
114656
|
+
const negs = [];
|
|
114657
|
+
let i = pos + 1;
|
|
114658
|
+
let sawStart = false;
|
|
114659
|
+
let uflag = false;
|
|
114660
|
+
let escaping = false;
|
|
114661
|
+
let negate = false;
|
|
114662
|
+
let endPos = pos;
|
|
114663
|
+
let rangeStart = '';
|
|
114664
|
+
WHILE: while (i < glob.length) {
|
|
114665
|
+
const c = glob.charAt(i);
|
|
114666
|
+
if ((c === '!' || c === '^') && i === pos + 1) {
|
|
114667
|
+
negate = true;
|
|
114668
|
+
i++;
|
|
114669
|
+
continue;
|
|
114670
|
+
}
|
|
114671
|
+
if (c === ']' && sawStart && !escaping) {
|
|
114672
|
+
endPos = i + 1;
|
|
114673
|
+
break;
|
|
114674
|
+
}
|
|
114675
|
+
sawStart = true;
|
|
114676
|
+
if (c === '\\') {
|
|
114677
|
+
if (!escaping) {
|
|
114678
|
+
escaping = true;
|
|
114679
|
+
i++;
|
|
114680
|
+
continue;
|
|
114681
|
+
}
|
|
114682
|
+
// escaped \ char, fall through and treat like normal char
|
|
114683
|
+
}
|
|
114684
|
+
if (c === '[' && !escaping) {
|
|
114685
|
+
// either a posix class, a collation equivalent, or just a [
|
|
114686
|
+
for (const [cls, [unip, u, neg]] of Object.entries(posixClasses)) {
|
|
114687
|
+
if (glob.startsWith(cls, i)) {
|
|
114688
|
+
// invalid, [a-[] is fine, but not [a-[:alpha]]
|
|
114689
|
+
if (rangeStart) {
|
|
114690
|
+
return ['$.', false, glob.length - pos, true];
|
|
114691
|
+
}
|
|
114692
|
+
i += cls.length;
|
|
114693
|
+
if (neg)
|
|
114694
|
+
negs.push(unip);
|
|
114695
|
+
else
|
|
114696
|
+
ranges.push(unip);
|
|
114697
|
+
uflag = uflag || u;
|
|
114698
|
+
continue WHILE;
|
|
114699
|
+
}
|
|
114700
|
+
}
|
|
114701
|
+
}
|
|
114702
|
+
// now it's just a normal character, effectively
|
|
114703
|
+
escaping = false;
|
|
114704
|
+
if (rangeStart) {
|
|
114705
|
+
// throw this range away if it's not valid, but others
|
|
114706
|
+
// can still match.
|
|
114707
|
+
if (c > rangeStart) {
|
|
114708
|
+
ranges.push(braceEscape(rangeStart) + '-' + braceEscape(c));
|
|
114709
|
+
}
|
|
114710
|
+
else if (c === rangeStart) {
|
|
114711
|
+
ranges.push(braceEscape(c));
|
|
114712
|
+
}
|
|
114713
|
+
rangeStart = '';
|
|
114714
|
+
i++;
|
|
114715
|
+
continue;
|
|
114716
|
+
}
|
|
114717
|
+
// now might be the start of a range.
|
|
114718
|
+
// can be either c-d or c-] or c<more...>] or c] at this point
|
|
114719
|
+
if (glob.startsWith('-]', i + 1)) {
|
|
114720
|
+
ranges.push(braceEscape(c + '-'));
|
|
114721
|
+
i += 2;
|
|
114722
|
+
continue;
|
|
114723
|
+
}
|
|
114724
|
+
if (glob.startsWith('-', i + 1)) {
|
|
114725
|
+
rangeStart = c;
|
|
114726
|
+
i += 2;
|
|
114727
|
+
continue;
|
|
114728
|
+
}
|
|
114729
|
+
// not the start of a range, just a single character
|
|
114730
|
+
ranges.push(braceEscape(c));
|
|
114731
|
+
i++;
|
|
114732
|
+
}
|
|
114733
|
+
if (endPos < i) {
|
|
114734
|
+
// didn't see the end of the class, not a valid class,
|
|
114735
|
+
// but might still be valid as a literal match.
|
|
114736
|
+
return ['', false, 0, false];
|
|
114737
|
+
}
|
|
114738
|
+
// if we got no ranges and no negates, then we have a range that
|
|
114739
|
+
// cannot possibly match anything, and that poisons the whole glob
|
|
114740
|
+
if (!ranges.length && !negs.length) {
|
|
114741
|
+
return ['$.', false, glob.length - pos, true];
|
|
114742
|
+
}
|
|
114743
|
+
// if we got one positive range, and it's a single character, then that's
|
|
114744
|
+
// not actually a magic pattern, it's just that one literal character.
|
|
114745
|
+
// we should not treat that as "magic", we should just return the literal
|
|
114746
|
+
// character. [_] is a perfectly valid way to escape glob magic chars.
|
|
114747
|
+
if (negs.length === 0 &&
|
|
114748
|
+
ranges.length === 1 &&
|
|
114749
|
+
/^\\?.$/.test(ranges[0]) &&
|
|
114750
|
+
!negate) {
|
|
114751
|
+
const r = ranges[0].length === 2 ? ranges[0].slice(-1) : ranges[0];
|
|
114752
|
+
return [regexpEscape(r), false, endPos - pos, false];
|
|
114753
|
+
}
|
|
114754
|
+
const sranges = '[' + (negate ? '^' : '') + rangesToString(ranges) + ']';
|
|
114755
|
+
const snegs = '[' + (negate ? '' : '^') + rangesToString(negs) + ']';
|
|
114756
|
+
const comb = ranges.length && negs.length ? '(' + sranges + '|' + snegs + ')'
|
|
114757
|
+
: ranges.length ? sranges
|
|
114758
|
+
: snegs;
|
|
114759
|
+
return [comb, uflag, endPos - pos, true];
|
|
114760
|
+
};
|
|
114761
|
+
|
|
114762
|
+
/**
|
|
114763
|
+
* Un-escape a string that has been escaped with {@link escape}.
|
|
114764
|
+
*
|
|
114765
|
+
* If the {@link MinimatchOptions.windowsPathsNoEscape} option is used, then
|
|
114766
|
+
* square-bracket escapes are removed, but not backslash escapes.
|
|
114767
|
+
*
|
|
114768
|
+
* For example, it will turn the string `'[*]'` into `*`, but it will not
|
|
114769
|
+
* turn `'\\*'` into `'*'`, because `\` is a path separator in
|
|
114770
|
+
* `windowsPathsNoEscape` mode.
|
|
114771
|
+
*
|
|
114772
|
+
* When `windowsPathsNoEscape` is not set, then both square-bracket escapes and
|
|
114773
|
+
* backslash escapes are removed.
|
|
114774
|
+
*
|
|
114775
|
+
* Slashes (and backslashes in `windowsPathsNoEscape` mode) cannot be escaped
|
|
114776
|
+
* or unescaped.
|
|
114777
|
+
*
|
|
114778
|
+
* When `magicalBraces` is not set, escapes of braces (`{` and `}`) will not be
|
|
114779
|
+
* unescaped.
|
|
114780
|
+
*/
|
|
114781
|
+
const unescape$1 = (s, { windowsPathsNoEscape = false, magicalBraces = true, } = {}) => {
|
|
114782
|
+
if (magicalBraces) {
|
|
114783
|
+
return windowsPathsNoEscape ?
|
|
114784
|
+
s.replace(/\[([^/\\])\]/g, '$1')
|
|
114785
|
+
: s
|
|
114786
|
+
.replace(/((?!\\).|^)\[([^/\\])\]/g, '$1$2')
|
|
114787
|
+
.replace(/\\([^/])/g, '$1');
|
|
114788
|
+
}
|
|
114789
|
+
return windowsPathsNoEscape ?
|
|
114790
|
+
s.replace(/\[([^/\\{}])\]/g, '$1')
|
|
114791
|
+
: s
|
|
114792
|
+
.replace(/((?!\\).|^)\[([^/\\{}])\]/g, '$1$2')
|
|
114793
|
+
.replace(/\\([^/{}])/g, '$1');
|
|
114794
|
+
};
|
|
114795
|
+
|
|
114796
|
+
// parse a single path portion
|
|
114797
|
+
var _a;
|
|
114798
|
+
const types$2 = new Set(['!', '?', '+', '*', '@']);
|
|
114799
|
+
const isExtglobType = (c) => types$2.has(c);
|
|
114800
|
+
const isExtglobAST = (c) => isExtglobType(c.type);
|
|
114801
|
+
// Map of which extglob types can adopt the children of a nested extglob
|
|
114802
|
+
//
|
|
114803
|
+
// anything but ! can adopt a matching type:
|
|
114804
|
+
// +(a|+(b|c)|d) => +(a|b|c|d)
|
|
114805
|
+
// *(a|*(b|c)|d) => *(a|b|c|d)
|
|
114806
|
+
// @(a|@(b|c)|d) => @(a|b|c|d)
|
|
114807
|
+
// ?(a|?(b|c)|d) => ?(a|b|c|d)
|
|
114808
|
+
//
|
|
114809
|
+
// * can adopt anything, because 0 or repetition is allowed
|
|
114810
|
+
// *(a|?(b|c)|d) => *(a|b|c|d)
|
|
114811
|
+
// *(a|+(b|c)|d) => *(a|b|c|d)
|
|
114812
|
+
// *(a|@(b|c)|d) => *(a|b|c|d)
|
|
114813
|
+
//
|
|
114814
|
+
// + can adopt @, because 1 or repetition is allowed
|
|
114815
|
+
// +(a|@(b|c)|d) => +(a|b|c|d)
|
|
114816
|
+
//
|
|
114817
|
+
// + and @ CANNOT adopt *, because 0 would be allowed
|
|
114818
|
+
// +(a|*(b|c)|d) => would match "", on *(b|c)
|
|
114819
|
+
// @(a|*(b|c)|d) => would match "", on *(b|c)
|
|
114820
|
+
//
|
|
114821
|
+
// + and @ CANNOT adopt ?, because 0 would be allowed
|
|
114822
|
+
// +(a|?(b|c)|d) => would match "", on ?(b|c)
|
|
114823
|
+
// @(a|?(b|c)|d) => would match "", on ?(b|c)
|
|
114824
|
+
//
|
|
114825
|
+
// ? can adopt @, because 0 or 1 is allowed
|
|
114826
|
+
// ?(a|@(b|c)|d) => ?(a|b|c|d)
|
|
114827
|
+
//
|
|
114828
|
+
// ? and @ CANNOT adopt * or +, because >1 would be allowed
|
|
114829
|
+
// ?(a|*(b|c)|d) => would match bbb on *(b|c)
|
|
114830
|
+
// @(a|*(b|c)|d) => would match bbb on *(b|c)
|
|
114831
|
+
// ?(a|+(b|c)|d) => would match bbb on +(b|c)
|
|
114832
|
+
// @(a|+(b|c)|d) => would match bbb on +(b|c)
|
|
114833
|
+
//
|
|
113582
114834
|
// ! CANNOT adopt ! (nothing else can either)
|
|
113583
114835
|
// !(a|!(b|c)|d) => !(a|b|c|d) would fail to match on b (not not b|c)
|
|
113584
114836
|
//
|
|
@@ -120950,1568 +122202,511 @@ class GlobUtil {
|
|
|
120950
122202
|
this.matchSync(m, absolute, ifDir);
|
|
120951
122203
|
}
|
|
120952
122204
|
for (const t of processor.subwalkTargets()) {
|
|
120953
|
-
if (this.maxDepth !== Infinity && t.depth() >= this.maxDepth) {
|
|
120954
|
-
continue;
|
|
120955
|
-
}
|
|
120956
|
-
tasks++;
|
|
120957
|
-
const children = t.readdirSync();
|
|
120958
|
-
this.walkCB3Sync(t, children, processor, next);
|
|
120959
|
-
}
|
|
120960
|
-
next();
|
|
120961
|
-
}
|
|
120962
|
-
walkCB3Sync(target, entries, processor, cb) {
|
|
120963
|
-
processor = processor.filterEntries(target, entries);
|
|
120964
|
-
let tasks = 1;
|
|
120965
|
-
const next = () => {
|
|
120966
|
-
if (--tasks === 0)
|
|
120967
|
-
cb();
|
|
120968
|
-
};
|
|
120969
|
-
for (const [m, absolute, ifDir] of processor.matches.entries()) {
|
|
120970
|
-
if (this.#ignored(m))
|
|
120971
|
-
continue;
|
|
120972
|
-
this.matchSync(m, absolute, ifDir);
|
|
120973
|
-
}
|
|
120974
|
-
for (const [target, patterns] of processor.subwalks.entries()) {
|
|
120975
|
-
tasks++;
|
|
120976
|
-
this.walkCB2Sync(target, patterns, processor.child(), next);
|
|
120977
|
-
}
|
|
120978
|
-
next();
|
|
120979
|
-
}
|
|
120980
|
-
}
|
|
120981
|
-
class GlobWalker extends GlobUtil {
|
|
120982
|
-
matches = new Set();
|
|
120983
|
-
constructor(patterns, path, opts) {
|
|
120984
|
-
super(patterns, path, opts);
|
|
120985
|
-
}
|
|
120986
|
-
matchEmit(e) {
|
|
120987
|
-
this.matches.add(e);
|
|
120988
|
-
}
|
|
120989
|
-
async walk() {
|
|
120990
|
-
if (this.signal?.aborted)
|
|
120991
|
-
throw this.signal.reason;
|
|
120992
|
-
if (this.path.isUnknown()) {
|
|
120993
|
-
await this.path.lstat();
|
|
120994
|
-
}
|
|
120995
|
-
await new Promise((res, rej) => {
|
|
120996
|
-
this.walkCB(this.path, this.patterns, () => {
|
|
120997
|
-
if (this.signal?.aborted) {
|
|
120998
|
-
rej(this.signal.reason);
|
|
120999
|
-
}
|
|
121000
|
-
else {
|
|
121001
|
-
res(this.matches);
|
|
121002
|
-
}
|
|
121003
|
-
});
|
|
121004
|
-
});
|
|
121005
|
-
return this.matches;
|
|
121006
|
-
}
|
|
121007
|
-
walkSync() {
|
|
121008
|
-
if (this.signal?.aborted)
|
|
121009
|
-
throw this.signal.reason;
|
|
121010
|
-
if (this.path.isUnknown()) {
|
|
121011
|
-
this.path.lstatSync();
|
|
121012
|
-
}
|
|
121013
|
-
// nothing for the callback to do, because this never pauses
|
|
121014
|
-
this.walkCBSync(this.path, this.patterns, () => {
|
|
121015
|
-
if (this.signal?.aborted)
|
|
121016
|
-
throw this.signal.reason;
|
|
121017
|
-
});
|
|
121018
|
-
return this.matches;
|
|
121019
|
-
}
|
|
121020
|
-
}
|
|
121021
|
-
class GlobStream extends GlobUtil {
|
|
121022
|
-
results;
|
|
121023
|
-
constructor(patterns, path, opts) {
|
|
121024
|
-
super(patterns, path, opts);
|
|
121025
|
-
this.results = new Minipass({
|
|
121026
|
-
signal: this.signal,
|
|
121027
|
-
objectMode: true,
|
|
121028
|
-
});
|
|
121029
|
-
this.results.on('drain', () => this.resume());
|
|
121030
|
-
this.results.on('resume', () => this.resume());
|
|
121031
|
-
}
|
|
121032
|
-
matchEmit(e) {
|
|
121033
|
-
this.results.write(e);
|
|
121034
|
-
if (!this.results.flowing)
|
|
121035
|
-
this.pause();
|
|
121036
|
-
}
|
|
121037
|
-
stream() {
|
|
121038
|
-
const target = this.path;
|
|
121039
|
-
if (target.isUnknown()) {
|
|
121040
|
-
target.lstat().then(() => {
|
|
121041
|
-
this.walkCB(target, this.patterns, () => this.results.end());
|
|
121042
|
-
});
|
|
121043
|
-
}
|
|
121044
|
-
else {
|
|
121045
|
-
this.walkCB(target, this.patterns, () => this.results.end());
|
|
121046
|
-
}
|
|
121047
|
-
return this.results;
|
|
121048
|
-
}
|
|
121049
|
-
streamSync() {
|
|
121050
|
-
if (this.path.isUnknown()) {
|
|
121051
|
-
this.path.lstatSync();
|
|
121052
|
-
}
|
|
121053
|
-
this.walkCBSync(this.path, this.patterns, () => this.results.end());
|
|
121054
|
-
return this.results;
|
|
121055
|
-
}
|
|
121056
|
-
}
|
|
121057
|
-
|
|
121058
|
-
// if no process global, just call it linux.
|
|
121059
|
-
// so we default to case-sensitive, / separators
|
|
121060
|
-
const defaultPlatform = (typeof process === 'object' &&
|
|
121061
|
-
process &&
|
|
121062
|
-
typeof process.platform === 'string') ?
|
|
121063
|
-
process.platform
|
|
121064
|
-
: 'linux';
|
|
121065
|
-
/**
|
|
121066
|
-
* An object that can perform glob pattern traversals.
|
|
121067
|
-
*/
|
|
121068
|
-
class Glob {
|
|
121069
|
-
absolute;
|
|
121070
|
-
cwd;
|
|
121071
|
-
root;
|
|
121072
|
-
dot;
|
|
121073
|
-
dotRelative;
|
|
121074
|
-
follow;
|
|
121075
|
-
ignore;
|
|
121076
|
-
magicalBraces;
|
|
121077
|
-
mark;
|
|
121078
|
-
matchBase;
|
|
121079
|
-
maxDepth;
|
|
121080
|
-
nobrace;
|
|
121081
|
-
nocase;
|
|
121082
|
-
nodir;
|
|
121083
|
-
noext;
|
|
121084
|
-
noglobstar;
|
|
121085
|
-
pattern;
|
|
121086
|
-
platform;
|
|
121087
|
-
realpath;
|
|
121088
|
-
scurry;
|
|
121089
|
-
stat;
|
|
121090
|
-
signal;
|
|
121091
|
-
windowsPathsNoEscape;
|
|
121092
|
-
withFileTypes;
|
|
121093
|
-
includeChildMatches;
|
|
121094
|
-
/**
|
|
121095
|
-
* The options provided to the constructor.
|
|
121096
|
-
*/
|
|
121097
|
-
opts;
|
|
121098
|
-
/**
|
|
121099
|
-
* An array of parsed immutable {@link Pattern} objects.
|
|
121100
|
-
*/
|
|
121101
|
-
patterns;
|
|
121102
|
-
/**
|
|
121103
|
-
* All options are stored as properties on the `Glob` object.
|
|
121104
|
-
*
|
|
121105
|
-
* See {@link GlobOptions} for full options descriptions.
|
|
121106
|
-
*
|
|
121107
|
-
* Note that a previous `Glob` object can be passed as the
|
|
121108
|
-
* `GlobOptions` to another `Glob` instantiation to re-use settings
|
|
121109
|
-
* and caches with a new pattern.
|
|
121110
|
-
*
|
|
121111
|
-
* Traversal functions can be called multiple times to run the walk
|
|
121112
|
-
* again.
|
|
121113
|
-
*/
|
|
121114
|
-
constructor(pattern, opts) {
|
|
121115
|
-
/* c8 ignore start */
|
|
121116
|
-
if (!opts)
|
|
121117
|
-
throw new TypeError('glob options required');
|
|
121118
|
-
/* c8 ignore stop */
|
|
121119
|
-
this.withFileTypes = !!opts.withFileTypes;
|
|
121120
|
-
this.signal = opts.signal;
|
|
121121
|
-
this.follow = !!opts.follow;
|
|
121122
|
-
this.dot = !!opts.dot;
|
|
121123
|
-
this.dotRelative = !!opts.dotRelative;
|
|
121124
|
-
this.nodir = !!opts.nodir;
|
|
121125
|
-
this.mark = !!opts.mark;
|
|
121126
|
-
if (!opts.cwd) {
|
|
121127
|
-
this.cwd = '';
|
|
121128
|
-
}
|
|
121129
|
-
else if (opts.cwd instanceof URL || opts.cwd.startsWith('file://')) {
|
|
121130
|
-
opts.cwd = fileURLToPath(opts.cwd);
|
|
121131
|
-
}
|
|
121132
|
-
this.cwd = opts.cwd || '';
|
|
121133
|
-
this.root = opts.root;
|
|
121134
|
-
this.magicalBraces = !!opts.magicalBraces;
|
|
121135
|
-
this.nobrace = !!opts.nobrace;
|
|
121136
|
-
this.noext = !!opts.noext;
|
|
121137
|
-
this.realpath = !!opts.realpath;
|
|
121138
|
-
this.absolute = opts.absolute;
|
|
121139
|
-
this.includeChildMatches = opts.includeChildMatches !== false;
|
|
121140
|
-
this.noglobstar = !!opts.noglobstar;
|
|
121141
|
-
this.matchBase = !!opts.matchBase;
|
|
121142
|
-
this.maxDepth =
|
|
121143
|
-
typeof opts.maxDepth === 'number' ? opts.maxDepth : Infinity;
|
|
121144
|
-
this.stat = !!opts.stat;
|
|
121145
|
-
this.ignore = opts.ignore;
|
|
121146
|
-
if (this.withFileTypes && this.absolute !== undefined) {
|
|
121147
|
-
throw new Error('cannot set absolute and withFileTypes:true');
|
|
121148
|
-
}
|
|
121149
|
-
if (typeof pattern === 'string') {
|
|
121150
|
-
pattern = [pattern];
|
|
121151
|
-
}
|
|
121152
|
-
this.windowsPathsNoEscape =
|
|
121153
|
-
!!opts.windowsPathsNoEscape ||
|
|
121154
|
-
opts.allowWindowsEscape ===
|
|
121155
|
-
false;
|
|
121156
|
-
if (this.windowsPathsNoEscape) {
|
|
121157
|
-
pattern = pattern.map(p => p.replace(/\\/g, '/'));
|
|
121158
|
-
}
|
|
121159
|
-
if (this.matchBase) {
|
|
121160
|
-
if (opts.noglobstar) {
|
|
121161
|
-
throw new TypeError('base matching requires globstar');
|
|
121162
|
-
}
|
|
121163
|
-
pattern = pattern.map(p => (p.includes('/') ? p : `./**/${p}`));
|
|
121164
|
-
}
|
|
121165
|
-
this.pattern = pattern;
|
|
121166
|
-
this.platform = opts.platform || defaultPlatform;
|
|
121167
|
-
this.opts = { ...opts, platform: this.platform };
|
|
121168
|
-
if (opts.scurry) {
|
|
121169
|
-
this.scurry = opts.scurry;
|
|
121170
|
-
if (opts.nocase !== undefined &&
|
|
121171
|
-
opts.nocase !== opts.scurry.nocase) {
|
|
121172
|
-
throw new Error('nocase option contradicts provided scurry option');
|
|
121173
|
-
}
|
|
121174
|
-
}
|
|
121175
|
-
else {
|
|
121176
|
-
const Scurry = opts.platform === 'win32' ? PathScurryWin32
|
|
121177
|
-
: opts.platform === 'darwin' ? PathScurryDarwin
|
|
121178
|
-
: opts.platform ? PathScurryPosix
|
|
121179
|
-
: PathScurry;
|
|
121180
|
-
this.scurry = new Scurry(this.cwd, {
|
|
121181
|
-
nocase: opts.nocase,
|
|
121182
|
-
fs: opts.fs,
|
|
121183
|
-
});
|
|
121184
|
-
}
|
|
121185
|
-
this.nocase = this.scurry.nocase;
|
|
121186
|
-
// If you do nocase:true on a case-sensitive file system, then
|
|
121187
|
-
// we need to use regexps instead of strings for non-magic
|
|
121188
|
-
// path portions, because statting `aBc` won't return results
|
|
121189
|
-
// for the file `AbC` for example.
|
|
121190
|
-
const nocaseMagicOnly = this.platform === 'darwin' || this.platform === 'win32';
|
|
121191
|
-
const mmo = {
|
|
121192
|
-
// default nocase based on platform
|
|
121193
|
-
...opts,
|
|
121194
|
-
dot: this.dot,
|
|
121195
|
-
matchBase: this.matchBase,
|
|
121196
|
-
nobrace: this.nobrace,
|
|
121197
|
-
nocase: this.nocase,
|
|
121198
|
-
nocaseMagicOnly,
|
|
121199
|
-
nocomment: true,
|
|
121200
|
-
noext: this.noext,
|
|
121201
|
-
nonegate: true,
|
|
121202
|
-
optimizationLevel: 2,
|
|
121203
|
-
platform: this.platform,
|
|
121204
|
-
windowsPathsNoEscape: this.windowsPathsNoEscape,
|
|
121205
|
-
debug: !!this.opts.debug,
|
|
121206
|
-
};
|
|
121207
|
-
const mms = this.pattern.map(p => new Minimatch(p, mmo));
|
|
121208
|
-
const [matchSet, globParts] = mms.reduce((set, m) => {
|
|
121209
|
-
set[0].push(...m.set);
|
|
121210
|
-
set[1].push(...m.globParts);
|
|
121211
|
-
return set;
|
|
121212
|
-
}, [[], []]);
|
|
121213
|
-
this.patterns = matchSet.map((set, i) => {
|
|
121214
|
-
const g = globParts[i];
|
|
121215
|
-
/* c8 ignore start */
|
|
121216
|
-
if (!g)
|
|
121217
|
-
throw new Error('invalid pattern object');
|
|
121218
|
-
/* c8 ignore stop */
|
|
121219
|
-
return new Pattern(set, g, 0, this.platform);
|
|
121220
|
-
});
|
|
121221
|
-
}
|
|
121222
|
-
async walk() {
|
|
121223
|
-
// Walkers always return array of Path objects, so we just have to
|
|
121224
|
-
// coerce them into the right shape. It will have already called
|
|
121225
|
-
// realpath() if the option was set to do so, so we know that's cached.
|
|
121226
|
-
// start out knowing the cwd, at least
|
|
121227
|
-
return [
|
|
121228
|
-
...(await new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
121229
|
-
...this.opts,
|
|
121230
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121231
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121232
|
-
: Infinity,
|
|
121233
|
-
platform: this.platform,
|
|
121234
|
-
nocase: this.nocase,
|
|
121235
|
-
includeChildMatches: this.includeChildMatches,
|
|
121236
|
-
}).walk()),
|
|
121237
|
-
];
|
|
121238
|
-
}
|
|
121239
|
-
walkSync() {
|
|
121240
|
-
return [
|
|
121241
|
-
...new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
121242
|
-
...this.opts,
|
|
121243
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121244
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121245
|
-
: Infinity,
|
|
121246
|
-
platform: this.platform,
|
|
121247
|
-
nocase: this.nocase,
|
|
121248
|
-
includeChildMatches: this.includeChildMatches,
|
|
121249
|
-
}).walkSync(),
|
|
121250
|
-
];
|
|
121251
|
-
}
|
|
121252
|
-
stream() {
|
|
121253
|
-
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
121254
|
-
...this.opts,
|
|
121255
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121256
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121257
|
-
: Infinity,
|
|
121258
|
-
platform: this.platform,
|
|
121259
|
-
nocase: this.nocase,
|
|
121260
|
-
includeChildMatches: this.includeChildMatches,
|
|
121261
|
-
}).stream();
|
|
121262
|
-
}
|
|
121263
|
-
streamSync() {
|
|
121264
|
-
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
121265
|
-
...this.opts,
|
|
121266
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121267
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121268
|
-
: Infinity,
|
|
121269
|
-
platform: this.platform,
|
|
121270
|
-
nocase: this.nocase,
|
|
121271
|
-
includeChildMatches: this.includeChildMatches,
|
|
121272
|
-
}).streamSync();
|
|
121273
|
-
}
|
|
121274
|
-
/**
|
|
121275
|
-
* Default sync iteration function. Returns a Generator that
|
|
121276
|
-
* iterates over the results.
|
|
121277
|
-
*/
|
|
121278
|
-
iterateSync() {
|
|
121279
|
-
return this.streamSync()[Symbol.iterator]();
|
|
121280
|
-
}
|
|
121281
|
-
[Symbol.iterator]() {
|
|
121282
|
-
return this.iterateSync();
|
|
121283
|
-
}
|
|
121284
|
-
/**
|
|
121285
|
-
* Default async iteration function. Returns an AsyncGenerator that
|
|
121286
|
-
* iterates over the results.
|
|
121287
|
-
*/
|
|
121288
|
-
iterate() {
|
|
121289
|
-
return this.stream()[Symbol.asyncIterator]();
|
|
121290
|
-
}
|
|
121291
|
-
[Symbol.asyncIterator]() {
|
|
121292
|
-
return this.iterate();
|
|
121293
|
-
}
|
|
121294
|
-
}
|
|
121295
|
-
|
|
121296
|
-
/**
|
|
121297
|
-
* Return true if the patterns provided contain any magic glob characters,
|
|
121298
|
-
* given the options provided.
|
|
121299
|
-
*
|
|
121300
|
-
* Brace expansion is not considered "magic" unless the `magicalBraces` option
|
|
121301
|
-
* is set, as brace expansion just turns one string into an array of strings.
|
|
121302
|
-
* So a pattern like `'x{a,b}y'` would return `false`, because `'xay'` and
|
|
121303
|
-
* `'xby'` both do not contain any magic glob characters, and it's treated the
|
|
121304
|
-
* same as if you had called it on `['xay', 'xby']`. When `magicalBraces:true`
|
|
121305
|
-
* is in the options, brace expansion _is_ treated as a pattern having magic.
|
|
121306
|
-
*/
|
|
121307
|
-
const hasMagic = (pattern, options = {}) => {
|
|
121308
|
-
if (!Array.isArray(pattern)) {
|
|
121309
|
-
pattern = [pattern];
|
|
121310
|
-
}
|
|
121311
|
-
for (const p of pattern) {
|
|
121312
|
-
if (new Minimatch(p, options).hasMagic())
|
|
121313
|
-
return true;
|
|
121314
|
-
}
|
|
121315
|
-
return false;
|
|
121316
|
-
};
|
|
121317
|
-
|
|
121318
|
-
function globStreamSync(pattern, options = {}) {
|
|
121319
|
-
return new Glob(pattern, options).streamSync();
|
|
121320
|
-
}
|
|
121321
|
-
function globStream(pattern, options = {}) {
|
|
121322
|
-
return new Glob(pattern, options).stream();
|
|
121323
|
-
}
|
|
121324
|
-
function globSync(pattern, options = {}) {
|
|
121325
|
-
return new Glob(pattern, options).walkSync();
|
|
121326
|
-
}
|
|
121327
|
-
async function glob_(pattern, options = {}) {
|
|
121328
|
-
return new Glob(pattern, options).walk();
|
|
121329
|
-
}
|
|
121330
|
-
function globIterateSync(pattern, options = {}) {
|
|
121331
|
-
return new Glob(pattern, options).iterateSync();
|
|
121332
|
-
}
|
|
121333
|
-
function globIterate(pattern, options = {}) {
|
|
121334
|
-
return new Glob(pattern, options).iterate();
|
|
121335
|
-
}
|
|
121336
|
-
// aliases: glob.sync.stream() glob.stream.sync() glob.sync() etc
|
|
121337
|
-
const streamSync = globStreamSync;
|
|
121338
|
-
const stream = Object.assign(globStream, { sync: globStreamSync });
|
|
121339
|
-
const iterateSync = globIterateSync;
|
|
121340
|
-
const iterate = Object.assign(globIterate, {
|
|
121341
|
-
sync: globIterateSync,
|
|
121342
|
-
});
|
|
121343
|
-
const sync = Object.assign(globSync, {
|
|
121344
|
-
stream: globStreamSync,
|
|
121345
|
-
iterate: globIterateSync,
|
|
121346
|
-
});
|
|
121347
|
-
const glob = Object.assign(glob_, {
|
|
121348
|
-
glob: glob_,
|
|
121349
|
-
globSync,
|
|
121350
|
-
sync,
|
|
121351
|
-
globStream,
|
|
121352
|
-
stream,
|
|
121353
|
-
globStreamSync,
|
|
121354
|
-
streamSync,
|
|
121355
|
-
globIterate,
|
|
121356
|
-
iterate,
|
|
121357
|
-
globIterateSync,
|
|
121358
|
-
iterateSync,
|
|
121359
|
-
Glob,
|
|
121360
|
-
hasMagic,
|
|
121361
|
-
escape: escape$1,
|
|
121362
|
-
unescape: unescape$1,
|
|
121363
|
-
});
|
|
121364
|
-
glob.glob = glob;
|
|
121365
|
-
|
|
121366
|
-
function matchesQuantizationVariant({ filePath, variant }) {
|
|
121367
|
-
if (!variant) {
|
|
121368
|
-
return false;
|
|
121369
|
-
}
|
|
121370
|
-
const escapedVariant = variant.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
121371
|
-
const matcher = new RegExp(`(^|[\\-./_])${escapedVariant}(?=[\\-./]|$)`, "i");
|
|
121372
|
-
const normalizedPath = filePath.replace(/\\/g, "/");
|
|
121373
|
-
const segments = normalizedPath.split("/").filter(Boolean);
|
|
121374
|
-
if (segments.length === 0) {
|
|
121375
|
-
return false;
|
|
121376
|
-
}
|
|
121377
|
-
const filename = segments[segments.length - 1].replace(/\.gguf$/i, "");
|
|
121378
|
-
if (matcher.test(filename)) {
|
|
121379
|
-
return true;
|
|
121380
|
-
}
|
|
121381
|
-
return segments.slice(0, -1).some(segment => matcher.test(segment));
|
|
121382
|
-
}
|
|
121383
|
-
async function findQuantizedModelTarget({ model, path }) {
|
|
121384
|
-
if (model.source.type === "storage") {
|
|
121385
|
-
throw new Error("Model storage not supported yet");
|
|
121386
|
-
}
|
|
121387
|
-
if (model.format !== "gguf") {
|
|
121388
|
-
throw new Error(`Model format not supported: ${model.format}`);
|
|
121389
|
-
}
|
|
121390
|
-
const [, variant = null] = model.source.slug.split(":");
|
|
121391
|
-
const modelFiles = (await glob("**/*.gguf", {
|
|
121392
|
-
absolute: true,
|
|
121393
|
-
cwd: path,
|
|
121394
|
-
nodir: true
|
|
121395
|
-
})).filter(file => !/(mmproj|clip)/i.test(file));
|
|
121396
|
-
if (modelFiles.length <= 0) {
|
|
121397
|
-
throw new Error(`No models found for format: ${model.format}`);
|
|
121398
|
-
}
|
|
121399
|
-
modelFiles.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: "base" }));
|
|
121400
|
-
if (!variant) {
|
|
121401
|
-
return modelFiles[0];
|
|
121402
|
-
}
|
|
121403
|
-
const matches = modelFiles.filter(fileName => matchesQuantizationVariant({ filePath: fileName, variant: variant ?? "" }));
|
|
121404
|
-
if (matches.length === 0) {
|
|
121405
|
-
throw new Error(`No model found for format and variant: ${model.format} / ${variant}`);
|
|
121406
|
-
}
|
|
121407
|
-
return matches[0];
|
|
121408
|
-
}
|
|
121409
|
-
|
|
121410
|
-
const VLLM_START_ARGS = ["-m", "vllm.entrypoints.openai.api_server", "--host", "0.0.0.0"];
|
|
121411
|
-
const VLLM_EXECUTABLE = "python3";
|
|
121412
|
-
const DEFAULT_VLLM_CONTEXT_LENGTH = 2048;
|
|
121413
|
-
async function startVLLM({ enginePort, targetDirectory }) {
|
|
121414
|
-
const contextLength = Math.max(1, this.contextLength ?? DEFAULT_VLLM_CONTEXT_LENGTH);
|
|
121415
|
-
let modelPath = targetDirectory;
|
|
121416
|
-
if (this.model.format === "gguf") {
|
|
121417
|
-
modelPath = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
|
|
121418
|
-
}
|
|
121419
|
-
const engineConfig = this.engineConfig;
|
|
121420
|
-
const device = typeof engineConfig?.device === "string" ? engineConfig.device : process.env.VLLM_DEVICE;
|
|
121421
|
-
const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : process.env.VLLM_DTYPE;
|
|
121422
|
-
const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
|
|
121423
|
-
const args = [
|
|
121424
|
-
...VLLM_START_ARGS,
|
|
121425
|
-
"--port",
|
|
121426
|
-
String(enginePort),
|
|
121427
|
-
"--model",
|
|
121428
|
-
modelPath,
|
|
121429
|
-
"--served-model-name",
|
|
121430
|
-
SERVED_MODEL_NAME,
|
|
121431
|
-
"--max-model-len",
|
|
121432
|
-
String(contextLength),
|
|
121433
|
-
"--tensor-parallel-size",
|
|
121434
|
-
String(tensorParallelSize)
|
|
121435
|
-
];
|
|
121436
|
-
if (this.model.taskType === "embeddings") {
|
|
121437
|
-
args.push("--task", "embed");
|
|
121438
|
-
}
|
|
121439
|
-
if (device) {
|
|
121440
|
-
args.push("--device", device);
|
|
121441
|
-
}
|
|
121442
|
-
if (dtype) {
|
|
121443
|
-
args.push("--dtype", dtype);
|
|
121444
|
-
}
|
|
121445
|
-
args.push(...parseExtraArgs(engineConfig?.extraArgs));
|
|
121446
|
-
if (this.model.multimodalEnabled) {
|
|
121447
|
-
args.push("--limit-mm-per-prompt", process.env.VLLM_MM_LIMIT ?? '{"image":5}');
|
|
121448
|
-
}
|
|
121449
|
-
if (process.env.VLLM_TRUST_REMOTE_CODE === "true") {
|
|
121450
|
-
args.push("--trust-remote-code");
|
|
121451
|
-
}
|
|
121452
|
-
return createEngineProcess({ args, bin: VLLM_EXECUTABLE, logger: this.logger });
|
|
121453
|
-
}
|
|
121454
|
-
|
|
121455
|
-
// src/lib/cache-management.ts
|
|
121456
|
-
|
|
121457
|
-
// src/consts.ts
|
|
121458
|
-
var HUB_URL = "https://huggingface.co";
|
|
121459
|
-
|
|
121460
|
-
// src/error.ts
|
|
121461
|
-
async function createApiError(response, opts) {
|
|
121462
|
-
const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
|
|
121463
|
-
error.message = `Api error with status ${error.statusCode}${""}`;
|
|
121464
|
-
const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". ");
|
|
121465
|
-
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
|
|
121466
|
-
const json = await response.json();
|
|
121467
|
-
error.message = json.error || json.message || error.message;
|
|
121468
|
-
if (json.error_description) {
|
|
121469
|
-
error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description;
|
|
121470
|
-
}
|
|
121471
|
-
error.data = json;
|
|
121472
|
-
} else {
|
|
121473
|
-
error.data = { message: await response.text() };
|
|
121474
|
-
}
|
|
121475
|
-
error.message += `. ${trailer}`;
|
|
121476
|
-
throw error;
|
|
121477
|
-
}
|
|
121478
|
-
var HubApiError = class extends Error {
|
|
121479
|
-
statusCode;
|
|
121480
|
-
url;
|
|
121481
|
-
requestId;
|
|
121482
|
-
data;
|
|
121483
|
-
constructor(url, statusCode, requestId, message) {
|
|
121484
|
-
super(message);
|
|
121485
|
-
this.statusCode = statusCode;
|
|
121486
|
-
this.requestId = requestId;
|
|
121487
|
-
this.url = url;
|
|
121488
|
-
}
|
|
121489
|
-
};
|
|
121490
|
-
var InvalidApiResponseFormatError = class extends Error {
|
|
121491
|
-
};
|
|
121492
|
-
|
|
121493
|
-
// src/utils/checkCredentials.ts
|
|
121494
|
-
function checkAccessToken(accessToken) {
|
|
121495
|
-
if (!accessToken.startsWith("hf_")) {
|
|
121496
|
-
throw new TypeError("Your access token must start with 'hf_'");
|
|
121497
|
-
}
|
|
121498
|
-
}
|
|
121499
|
-
function checkCredentials(params) {
|
|
121500
|
-
if (params.accessToken) {
|
|
121501
|
-
checkAccessToken(params.accessToken);
|
|
121502
|
-
return params.accessToken;
|
|
121503
|
-
}
|
|
121504
|
-
if (params.credentials?.accessToken) {
|
|
121505
|
-
checkAccessToken(params.credentials.accessToken);
|
|
121506
|
-
return params.credentials.accessToken;
|
|
121507
|
-
}
|
|
121508
|
-
}
|
|
121509
|
-
|
|
121510
|
-
// src/utils/toRepoId.ts
|
|
121511
|
-
function toRepoId(repo) {
|
|
121512
|
-
if (typeof repo !== "string") {
|
|
121513
|
-
return repo;
|
|
121514
|
-
}
|
|
121515
|
-
if (repo.startsWith("model/") || repo.startsWith("models/")) {
|
|
121516
|
-
throw new TypeError(
|
|
121517
|
-
"A repo designation for a model should not start with 'models/', directly specify the model namespace / name"
|
|
121518
|
-
);
|
|
121519
|
-
}
|
|
121520
|
-
if (repo.startsWith("space/")) {
|
|
121521
|
-
throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'");
|
|
121522
|
-
}
|
|
121523
|
-
if (repo.startsWith("dataset/")) {
|
|
121524
|
-
throw new TypeError("Datasets should start with 'dataset/', plural, not 'dataset/'");
|
|
121525
|
-
}
|
|
121526
|
-
const slashes = repo.split("/").length - 1;
|
|
121527
|
-
if (repo.startsWith("spaces/")) {
|
|
121528
|
-
if (slashes !== 2) {
|
|
121529
|
-
throw new TypeError("Space Id must include namespace and name of the space");
|
|
121530
|
-
}
|
|
121531
|
-
return {
|
|
121532
|
-
type: "space",
|
|
121533
|
-
name: repo.slice("spaces/".length)
|
|
121534
|
-
};
|
|
121535
|
-
}
|
|
121536
|
-
if (repo.startsWith("datasets/")) {
|
|
121537
|
-
if (slashes > 2) {
|
|
121538
|
-
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
121539
|
-
}
|
|
121540
|
-
return {
|
|
121541
|
-
type: "dataset",
|
|
121542
|
-
name: repo.slice("datasets/".length)
|
|
121543
|
-
};
|
|
121544
|
-
}
|
|
121545
|
-
if (slashes > 1) {
|
|
121546
|
-
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
121547
|
-
}
|
|
121548
|
-
return {
|
|
121549
|
-
type: "model",
|
|
121550
|
-
name: repo
|
|
121551
|
-
};
|
|
121552
|
-
}
|
|
121553
|
-
new Promise((r) => {
|
|
121554
|
-
});
|
|
121555
|
-
|
|
121556
|
-
// src/utils/combineUint8Arrays.ts
|
|
121557
|
-
function combineUint8Arrays(a, b) {
|
|
121558
|
-
const aLength = a.length;
|
|
121559
|
-
const combinedBytes = new Uint8Array(aLength + b.length);
|
|
121560
|
-
combinedBytes.set(a);
|
|
121561
|
-
combinedBytes.set(b, aLength);
|
|
121562
|
-
return combinedBytes;
|
|
121563
|
-
}
|
|
121564
|
-
function readU64(b, n) {
|
|
121565
|
-
let x = 0;
|
|
121566
|
-
x |= b[n++] << 0;
|
|
121567
|
-
x |= b[n++] << 8;
|
|
121568
|
-
x |= b[n++] << 16;
|
|
121569
|
-
x |= b[n++] << 24;
|
|
121570
|
-
x |= b[n++] << 32;
|
|
121571
|
-
x |= b[n++] << 40;
|
|
121572
|
-
x |= b[n++] << 48;
|
|
121573
|
-
x |= b[n++] << 56;
|
|
121574
|
-
return x;
|
|
121575
|
-
}
|
|
121576
|
-
function readU32(b, n) {
|
|
121577
|
-
let x = 0;
|
|
121578
|
-
x |= b[n++] << 0;
|
|
121579
|
-
x |= b[n++] << 8;
|
|
121580
|
-
x |= b[n++] << 16;
|
|
121581
|
-
x |= b[n++] << 24;
|
|
121582
|
-
return x;
|
|
121583
|
-
}
|
|
121584
|
-
|
|
121585
|
-
// src/vendor/lz4js/index.ts
|
|
121586
|
-
var minMatch = 4;
|
|
121587
|
-
var hashSize = 1 << 16;
|
|
121588
|
-
makeHashTable();
|
|
121589
|
-
var magicNum = 407708164;
|
|
121590
|
-
var fdContentChksum = 4;
|
|
121591
|
-
var fdContentSize = 8;
|
|
121592
|
-
var fdBlockChksum = 16;
|
|
121593
|
-
var fdVersion = 64;
|
|
121594
|
-
var fdVersionMask = 192;
|
|
121595
|
-
var bsUncompressed = 2147483648;
|
|
121596
|
-
var bsShift = 4;
|
|
121597
|
-
var bsMask = 7;
|
|
121598
|
-
var bsMap = {
|
|
121599
|
-
4: 65536,
|
|
121600
|
-
5: 262144,
|
|
121601
|
-
6: 1048576,
|
|
121602
|
-
7: 4194304
|
|
121603
|
-
};
|
|
121604
|
-
function makeHashTable() {
|
|
121605
|
-
try {
|
|
121606
|
-
return new Uint32Array(hashSize);
|
|
121607
|
-
} catch (error) {
|
|
121608
|
-
const hashTable2 = new Array(hashSize);
|
|
121609
|
-
for (let i = 0; i < hashSize; i++) {
|
|
121610
|
-
hashTable2[i] = 0;
|
|
121611
|
-
}
|
|
121612
|
-
return hashTable2;
|
|
121613
|
-
}
|
|
121614
|
-
}
|
|
121615
|
-
function makeBuffer(size) {
|
|
121616
|
-
return new Uint8Array(size);
|
|
121617
|
-
}
|
|
121618
|
-
function sliceArray(array, start, end) {
|
|
121619
|
-
return array.slice(start, end);
|
|
121620
|
-
}
|
|
121621
|
-
function decompressBound(src) {
|
|
121622
|
-
let sIndex = 0;
|
|
121623
|
-
if (readU32(src, sIndex) !== magicNum) {
|
|
121624
|
-
throw new Error("invalid magic number");
|
|
121625
|
-
}
|
|
121626
|
-
sIndex += 4;
|
|
121627
|
-
const descriptor = src[sIndex++];
|
|
121628
|
-
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
121629
|
-
throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask));
|
|
121630
|
-
}
|
|
121631
|
-
const useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
121632
|
-
const useContentSize = (descriptor & fdContentSize) !== 0;
|
|
121633
|
-
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
121634
|
-
if (bsMap[bsIdx] === void 0) {
|
|
121635
|
-
throw new Error("invalid block size " + bsIdx);
|
|
121636
|
-
}
|
|
121637
|
-
const maxBlockSize = bsMap[bsIdx];
|
|
121638
|
-
if (useContentSize) {
|
|
121639
|
-
return readU64(src, sIndex);
|
|
121640
|
-
}
|
|
121641
|
-
sIndex++;
|
|
121642
|
-
let maxSize = 0;
|
|
121643
|
-
while (true) {
|
|
121644
|
-
let blockSize = readU32(src, sIndex);
|
|
121645
|
-
sIndex += 4;
|
|
121646
|
-
if (blockSize & bsUncompressed) {
|
|
121647
|
-
blockSize &= ~bsUncompressed;
|
|
121648
|
-
maxSize += blockSize;
|
|
121649
|
-
} else if (blockSize > 0) {
|
|
121650
|
-
maxSize += maxBlockSize;
|
|
121651
|
-
}
|
|
121652
|
-
if (blockSize === 0) {
|
|
121653
|
-
return maxSize;
|
|
121654
|
-
}
|
|
121655
|
-
if (useBlockSum) {
|
|
121656
|
-
sIndex += 4;
|
|
121657
|
-
}
|
|
121658
|
-
sIndex += blockSize;
|
|
121659
|
-
}
|
|
121660
|
-
}
|
|
121661
|
-
function decompressBlock(src, dst, sIndex, sLength, dIndex) {
|
|
121662
|
-
let mLength, mOffset, sEnd, n, i;
|
|
121663
|
-
const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0;
|
|
121664
|
-
sEnd = sIndex + sLength;
|
|
121665
|
-
while (sIndex < sEnd) {
|
|
121666
|
-
const token = src[sIndex++];
|
|
121667
|
-
let literalCount = token >> 4;
|
|
121668
|
-
if (literalCount > 0) {
|
|
121669
|
-
if (literalCount === 15) {
|
|
121670
|
-
while (true) {
|
|
121671
|
-
literalCount += src[sIndex];
|
|
121672
|
-
if (src[sIndex++] !== 255) {
|
|
121673
|
-
break;
|
|
121674
|
-
}
|
|
121675
|
-
}
|
|
121676
|
-
}
|
|
121677
|
-
for (n = sIndex + literalCount; sIndex < n; ) {
|
|
121678
|
-
dst[dIndex++] = src[sIndex++];
|
|
121679
|
-
}
|
|
121680
|
-
}
|
|
121681
|
-
if (sIndex >= sEnd) {
|
|
121682
|
-
break;
|
|
121683
|
-
}
|
|
121684
|
-
mLength = token & 15;
|
|
121685
|
-
mOffset = src[sIndex++] | src[sIndex++] << 8;
|
|
121686
|
-
if (mLength === 15) {
|
|
121687
|
-
while (true) {
|
|
121688
|
-
mLength += src[sIndex];
|
|
121689
|
-
if (src[sIndex++] !== 255) {
|
|
121690
|
-
break;
|
|
121691
|
-
}
|
|
121692
|
-
}
|
|
121693
|
-
}
|
|
121694
|
-
mLength += minMatch;
|
|
121695
|
-
if (hasCopyWithin && mOffset === 1) {
|
|
121696
|
-
dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength);
|
|
121697
|
-
dIndex += mLength;
|
|
121698
|
-
} else if (hasCopyWithin && mOffset > mLength && mLength > 31) {
|
|
121699
|
-
dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength);
|
|
121700
|
-
dIndex += mLength;
|
|
121701
|
-
} else {
|
|
121702
|
-
for (i = dIndex - mOffset, n = i + mLength; i < n; ) {
|
|
121703
|
-
dst[dIndex++] = dst[i++] | 0;
|
|
121704
|
-
}
|
|
121705
|
-
}
|
|
121706
|
-
}
|
|
121707
|
-
return dIndex;
|
|
121708
|
-
}
|
|
121709
|
-
function decompressFrame(src, dst) {
|
|
121710
|
-
let useBlockSum, useContentSum, useContentSize, descriptor;
|
|
121711
|
-
let sIndex = 0;
|
|
121712
|
-
let dIndex = 0;
|
|
121713
|
-
if (readU32(src, sIndex) !== magicNum) {
|
|
121714
|
-
throw new Error("invalid magic number");
|
|
121715
|
-
}
|
|
121716
|
-
sIndex += 4;
|
|
121717
|
-
descriptor = src[sIndex++];
|
|
121718
|
-
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
121719
|
-
throw new Error("incompatible descriptor version");
|
|
121720
|
-
}
|
|
121721
|
-
useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
121722
|
-
useContentSum = (descriptor & fdContentChksum) !== 0;
|
|
121723
|
-
useContentSize = (descriptor & fdContentSize) !== 0;
|
|
121724
|
-
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
121725
|
-
if (bsMap[bsIdx] === void 0) {
|
|
121726
|
-
throw new Error("invalid block size");
|
|
121727
|
-
}
|
|
121728
|
-
if (useContentSize) {
|
|
121729
|
-
sIndex += 8;
|
|
121730
|
-
}
|
|
121731
|
-
sIndex++;
|
|
121732
|
-
while (true) {
|
|
121733
|
-
var compSize;
|
|
121734
|
-
compSize = readU32(src, sIndex);
|
|
121735
|
-
sIndex += 4;
|
|
121736
|
-
if (compSize === 0) {
|
|
121737
|
-
break;
|
|
121738
|
-
}
|
|
121739
|
-
if (useBlockSum) {
|
|
121740
|
-
sIndex += 4;
|
|
121741
|
-
}
|
|
121742
|
-
if ((compSize & bsUncompressed) !== 0) {
|
|
121743
|
-
compSize &= ~bsUncompressed;
|
|
121744
|
-
for (let j = 0; j < compSize; j++) {
|
|
121745
|
-
dst[dIndex++] = src[sIndex++];
|
|
121746
|
-
}
|
|
121747
|
-
} else {
|
|
121748
|
-
dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex);
|
|
121749
|
-
sIndex += compSize;
|
|
121750
|
-
}
|
|
121751
|
-
}
|
|
121752
|
-
if (useContentSum) {
|
|
121753
|
-
sIndex += 4;
|
|
121754
|
-
}
|
|
121755
|
-
return dIndex;
|
|
121756
|
-
}
|
|
121757
|
-
function decompress(src, maxSize) {
|
|
121758
|
-
let dst, size;
|
|
121759
|
-
if (maxSize === void 0) {
|
|
121760
|
-
maxSize = decompressBound(src);
|
|
121761
|
-
}
|
|
121762
|
-
dst = makeBuffer(maxSize);
|
|
121763
|
-
size = decompressFrame(src, dst);
|
|
121764
|
-
if (size !== maxSize) {
|
|
121765
|
-
dst = sliceArray(dst, 0, size);
|
|
121766
|
-
}
|
|
121767
|
-
return dst;
|
|
121768
|
-
}
|
|
121769
|
-
|
|
121770
|
-
// src/utils/RangeList.ts
|
|
121771
|
-
var RangeList = class {
|
|
121772
|
-
ranges = [];
|
|
121773
|
-
/**
|
|
121774
|
-
* Add a range to the list. If it overlaps with existing ranges,
|
|
121775
|
-
* it will split them and increment reference counts accordingly.
|
|
121776
|
-
*/
|
|
121777
|
-
add(start, end) {
|
|
121778
|
-
if (end <= start) {
|
|
121779
|
-
throw new TypeError("End must be greater than start");
|
|
121780
|
-
}
|
|
121781
|
-
const overlappingRanges = [];
|
|
121782
|
-
for (let i = 0; i < this.ranges.length; i++) {
|
|
121783
|
-
const range2 = this.ranges[i];
|
|
121784
|
-
if (start < range2.end && end > range2.start) {
|
|
121785
|
-
overlappingRanges.push({ index: i, range: range2 });
|
|
121786
|
-
}
|
|
121787
|
-
if (range2.data !== null) {
|
|
121788
|
-
throw new Error("Overlapping range already has data");
|
|
121789
|
-
}
|
|
121790
|
-
}
|
|
121791
|
-
if (overlappingRanges.length === 0) {
|
|
121792
|
-
this.ranges.push({ start, end, refCount: 1, data: null });
|
|
121793
|
-
this.ranges.sort((a, b) => a.start - b.start);
|
|
121794
|
-
return;
|
|
121795
|
-
}
|
|
121796
|
-
const newRanges = [];
|
|
121797
|
-
let currentPos = start;
|
|
121798
|
-
for (let i = 0; i < overlappingRanges.length; i++) {
|
|
121799
|
-
const { range: range2 } = overlappingRanges[i];
|
|
121800
|
-
if (currentPos < range2.start) {
|
|
121801
|
-
newRanges.push({
|
|
121802
|
-
start: currentPos,
|
|
121803
|
-
end: range2.start,
|
|
121804
|
-
refCount: 1,
|
|
121805
|
-
data: null
|
|
121806
|
-
});
|
|
121807
|
-
} else if (range2.start < currentPos) {
|
|
121808
|
-
newRanges.push({
|
|
121809
|
-
start: range2.start,
|
|
121810
|
-
end: currentPos,
|
|
121811
|
-
refCount: range2.refCount,
|
|
121812
|
-
data: null
|
|
121813
|
-
});
|
|
121814
|
-
}
|
|
121815
|
-
newRanges.push({
|
|
121816
|
-
start: Math.max(currentPos, range2.start),
|
|
121817
|
-
end: Math.min(end, range2.end),
|
|
121818
|
-
refCount: range2.refCount + 1,
|
|
121819
|
-
data: null
|
|
121820
|
-
});
|
|
121821
|
-
if (range2.end > end) {
|
|
121822
|
-
newRanges.push({
|
|
121823
|
-
start: end,
|
|
121824
|
-
end: range2.end,
|
|
121825
|
-
refCount: range2.refCount,
|
|
121826
|
-
data: null
|
|
121827
|
-
});
|
|
121828
|
-
}
|
|
121829
|
-
currentPos = Math.max(currentPos, range2.end);
|
|
121830
|
-
}
|
|
121831
|
-
if (currentPos < end) {
|
|
121832
|
-
newRanges.push({
|
|
121833
|
-
start: currentPos,
|
|
121834
|
-
end,
|
|
121835
|
-
refCount: 1,
|
|
121836
|
-
data: null
|
|
121837
|
-
});
|
|
121838
|
-
}
|
|
121839
|
-
const firstIndex = overlappingRanges[0].index;
|
|
121840
|
-
const lastIndex = overlappingRanges[overlappingRanges.length - 1].index;
|
|
121841
|
-
this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges);
|
|
121842
|
-
this.ranges.sort((a, b) => a.start - b.start);
|
|
121843
|
-
}
|
|
121844
|
-
/**
|
|
121845
|
-
* Remove a range from the list. The range must start and end at existing boundaries.
|
|
121846
|
-
*/
|
|
121847
|
-
remove(start, end) {
|
|
121848
|
-
if (end <= start) {
|
|
121849
|
-
throw new TypeError("End must be greater than start");
|
|
121850
|
-
}
|
|
121851
|
-
const affectedRanges = [];
|
|
121852
|
-
for (let i = 0; i < this.ranges.length; i++) {
|
|
121853
|
-
const range2 = this.ranges[i];
|
|
121854
|
-
if (start < range2.end && end > range2.start) {
|
|
121855
|
-
affectedRanges.push({ index: i, range: range2 });
|
|
121856
|
-
}
|
|
121857
|
-
}
|
|
121858
|
-
if (affectedRanges.length === 0) {
|
|
121859
|
-
throw new Error("No ranges found to remove");
|
|
121860
|
-
}
|
|
121861
|
-
if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) {
|
|
121862
|
-
throw new Error("Range boundaries must match existing boundaries");
|
|
121863
|
-
}
|
|
121864
|
-
for (let i = 0; i < affectedRanges.length; i++) {
|
|
121865
|
-
const { range: range2 } = affectedRanges[i];
|
|
121866
|
-
range2.refCount--;
|
|
121867
|
-
}
|
|
121868
|
-
this.ranges = this.ranges.filter((range2) => range2.refCount > 0);
|
|
121869
|
-
}
|
|
121870
|
-
/**
|
|
121871
|
-
* Get all ranges within the specified boundaries.
|
|
121872
|
-
*/
|
|
121873
|
-
getRanges(start, end) {
|
|
121874
|
-
if (end <= start) {
|
|
121875
|
-
throw new TypeError("End must be greater than start");
|
|
121876
|
-
}
|
|
121877
|
-
return this.ranges.filter((range2) => start < range2.end && end > range2.start);
|
|
121878
|
-
}
|
|
121879
|
-
/**
|
|
121880
|
-
* Get all ranges in the list
|
|
121881
|
-
*/
|
|
121882
|
-
getAllRanges() {
|
|
121883
|
-
return [...this.ranges];
|
|
121884
|
-
}
|
|
121885
|
-
};
|
|
121886
|
-
|
|
121887
|
-
// src/utils/XetBlob.ts
|
|
121888
|
-
var JWT_SAFETY_PERIOD = 6e4;
|
|
121889
|
-
var JWT_CACHE_SIZE = 1e3;
|
|
121890
|
-
var compressionSchemeLabels = {
|
|
121891
|
-
[0 /* None */]: "None",
|
|
121892
|
-
[1 /* LZ4 */]: "LZ4",
|
|
121893
|
-
[2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4"
|
|
121894
|
-
};
|
|
121895
|
-
var XET_CHUNK_HEADER_BYTES = 8;
|
|
121896
|
-
var XetBlob = class extends Blob {
|
|
121897
|
-
fetch;
|
|
121898
|
-
accessToken;
|
|
121899
|
-
refreshUrl;
|
|
121900
|
-
reconstructionUrl;
|
|
121901
|
-
hash;
|
|
121902
|
-
start = 0;
|
|
121903
|
-
end = 0;
|
|
121904
|
-
internalLogging = false;
|
|
121905
|
-
reconstructionInfo;
|
|
121906
|
-
listener;
|
|
121907
|
-
constructor(params) {
|
|
121908
|
-
super([]);
|
|
121909
|
-
this.fetch = params.fetch ?? fetch.bind(globalThis);
|
|
121910
|
-
this.accessToken = checkCredentials(params);
|
|
121911
|
-
this.refreshUrl = params.refreshUrl;
|
|
121912
|
-
this.end = params.size;
|
|
121913
|
-
this.reconstructionUrl = params.reconstructionUrl;
|
|
121914
|
-
this.hash = params.hash;
|
|
121915
|
-
this.listener = params.listener;
|
|
121916
|
-
this.internalLogging = params.internalLogging ?? false;
|
|
121917
|
-
this.refreshUrl;
|
|
121918
|
-
}
|
|
121919
|
-
get size() {
|
|
121920
|
-
return this.end - this.start;
|
|
121921
|
-
}
|
|
121922
|
-
#clone() {
|
|
121923
|
-
const blob = new XetBlob({
|
|
121924
|
-
fetch: this.fetch,
|
|
121925
|
-
hash: this.hash,
|
|
121926
|
-
refreshUrl: this.refreshUrl,
|
|
121927
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
121928
|
-
reconstructionUrl: this.reconstructionUrl,
|
|
121929
|
-
size: this.size
|
|
121930
|
-
});
|
|
121931
|
-
blob.accessToken = this.accessToken;
|
|
121932
|
-
blob.start = this.start;
|
|
121933
|
-
blob.end = this.end;
|
|
121934
|
-
blob.reconstructionInfo = this.reconstructionInfo;
|
|
121935
|
-
blob.listener = this.listener;
|
|
121936
|
-
blob.internalLogging = this.internalLogging;
|
|
121937
|
-
return blob;
|
|
121938
|
-
}
|
|
121939
|
-
slice(start = 0, end = this.size) {
|
|
121940
|
-
const slice = this.#clone();
|
|
121941
|
-
slice.start = this.start + start;
|
|
121942
|
-
slice.end = Math.min(this.start + end, this.end);
|
|
121943
|
-
if (slice.start !== this.start || slice.end !== this.end) {
|
|
121944
|
-
slice.reconstructionInfo = void 0;
|
|
122205
|
+
if (this.maxDepth !== Infinity && t.depth() >= this.maxDepth) {
|
|
122206
|
+
continue;
|
|
122207
|
+
}
|
|
122208
|
+
tasks++;
|
|
122209
|
+
const children = t.readdirSync();
|
|
122210
|
+
this.walkCB3Sync(t, children, processor, next);
|
|
122211
|
+
}
|
|
122212
|
+
next();
|
|
121945
122213
|
}
|
|
121946
|
-
|
|
121947
|
-
|
|
121948
|
-
|
|
121949
|
-
|
|
121950
|
-
|
|
121951
|
-
|
|
122214
|
+
walkCB3Sync(target, entries, processor, cb) {
|
|
122215
|
+
processor = processor.filterEntries(target, entries);
|
|
122216
|
+
let tasks = 1;
|
|
122217
|
+
const next = () => {
|
|
122218
|
+
if (--tasks === 0)
|
|
122219
|
+
cb();
|
|
122220
|
+
};
|
|
122221
|
+
for (const [m, absolute, ifDir] of processor.matches.entries()) {
|
|
122222
|
+
if (this.#ignored(m))
|
|
122223
|
+
continue;
|
|
122224
|
+
this.matchSync(m, absolute, ifDir);
|
|
122225
|
+
}
|
|
122226
|
+
for (const [target, patterns] of processor.subwalks.entries()) {
|
|
122227
|
+
tasks++;
|
|
122228
|
+
this.walkCB2Sync(target, patterns, processor.child(), next);
|
|
122229
|
+
}
|
|
122230
|
+
next();
|
|
121952
122231
|
}
|
|
121953
|
-
|
|
121954
|
-
|
|
121955
|
-
|
|
121956
|
-
|
|
121957
|
-
|
|
121958
|
-
|
|
122232
|
+
}
|
|
122233
|
+
class GlobWalker extends GlobUtil {
|
|
122234
|
+
matches = new Set();
|
|
122235
|
+
constructor(patterns, path, opts) {
|
|
122236
|
+
super(patterns, path, opts);
|
|
122237
|
+
}
|
|
122238
|
+
matchEmit(e) {
|
|
122239
|
+
this.matches.add(e);
|
|
122240
|
+
}
|
|
122241
|
+
async walk() {
|
|
122242
|
+
if (this.signal?.aborted)
|
|
122243
|
+
throw this.signal.reason;
|
|
122244
|
+
if (this.path.isUnknown()) {
|
|
122245
|
+
await this.path.lstat();
|
|
121959
122246
|
}
|
|
121960
|
-
|
|
121961
|
-
|
|
121962
|
-
|
|
121963
|
-
|
|
121964
|
-
|
|
121965
|
-
|
|
121966
|
-
|
|
121967
|
-
|
|
121968
|
-
|
|
121969
|
-
|
|
121970
|
-
|
|
121971
|
-
await this.#loadReconstructionInfo();
|
|
122247
|
+
await new Promise((res, rej) => {
|
|
122248
|
+
this.walkCB(this.path, this.patterns, () => {
|
|
122249
|
+
if (this.signal?.aborted) {
|
|
122250
|
+
rej(this.signal.reason);
|
|
122251
|
+
}
|
|
122252
|
+
else {
|
|
122253
|
+
res(this.matches);
|
|
122254
|
+
}
|
|
122255
|
+
});
|
|
122256
|
+
});
|
|
122257
|
+
return this.matches;
|
|
121972
122258
|
}
|
|
121973
|
-
|
|
121974
|
-
|
|
121975
|
-
|
|
122259
|
+
walkSync() {
|
|
122260
|
+
if (this.signal?.aborted)
|
|
122261
|
+
throw this.signal.reason;
|
|
122262
|
+
if (this.path.isUnknown()) {
|
|
122263
|
+
this.path.lstatSync();
|
|
122264
|
+
}
|
|
122265
|
+
// nothing for the callback to do, because this never pauses
|
|
122266
|
+
this.walkCBSync(this.path, this.patterns, () => {
|
|
122267
|
+
if (this.signal?.aborted)
|
|
122268
|
+
throw this.signal.reason;
|
|
122269
|
+
});
|
|
122270
|
+
return this.matches;
|
|
121976
122271
|
}
|
|
121977
|
-
|
|
121978
|
-
|
|
121979
|
-
|
|
121980
|
-
|
|
121981
|
-
|
|
121982
|
-
|
|
121983
|
-
|
|
122272
|
+
}
|
|
122273
|
+
class GlobStream extends GlobUtil {
|
|
122274
|
+
results;
|
|
122275
|
+
constructor(patterns, path, opts) {
|
|
122276
|
+
super(patterns, path, opts);
|
|
122277
|
+
this.results = new Minipass({
|
|
122278
|
+
signal: this.signal,
|
|
122279
|
+
objectMode: true,
|
|
122280
|
+
});
|
|
122281
|
+
this.results.on('drain', () => this.resume());
|
|
122282
|
+
this.results.on('resume', () => this.resume());
|
|
121984
122283
|
}
|
|
121985
|
-
|
|
121986
|
-
|
|
121987
|
-
|
|
121988
|
-
|
|
121989
|
-
|
|
121990
|
-
|
|
121991
|
-
|
|
121992
|
-
if (
|
|
121993
|
-
|
|
122284
|
+
matchEmit(e) {
|
|
122285
|
+
this.results.write(e);
|
|
122286
|
+
if (!this.results.flowing)
|
|
122287
|
+
this.pause();
|
|
122288
|
+
}
|
|
122289
|
+
stream() {
|
|
122290
|
+
const target = this.path;
|
|
122291
|
+
if (target.isUnknown()) {
|
|
122292
|
+
target.lstat().then(() => {
|
|
122293
|
+
this.walkCB(target, this.patterns, () => this.results.end());
|
|
122294
|
+
});
|
|
121994
122295
|
}
|
|
121995
|
-
|
|
121996
|
-
|
|
121997
|
-
throw new Error(`Failed to find range list for term ${term.hash}`);
|
|
122296
|
+
else {
|
|
122297
|
+
this.walkCB(target, this.patterns, () => this.results.end());
|
|
121998
122298
|
}
|
|
121999
|
-
|
|
122000
|
-
|
|
122001
|
-
|
|
122002
|
-
|
|
122003
|
-
|
|
122004
|
-
for (const range2 of termRanges) {
|
|
122005
|
-
for (let chunk2 of range2.data) {
|
|
122006
|
-
if (readBytesToSkip) {
|
|
122007
|
-
const skipped = Math.min(readBytesToSkip, chunk2.byteLength);
|
|
122008
|
-
chunk2 = chunk2.slice(skipped);
|
|
122009
|
-
readBytesToSkip -= skipped;
|
|
122010
|
-
if (!chunk2.byteLength) {
|
|
122011
|
-
continue;
|
|
122012
|
-
}
|
|
122013
|
-
}
|
|
122014
|
-
if (chunk2.byteLength > maxBytes - totalBytesRead) {
|
|
122015
|
-
chunk2 = chunk2.slice(0, maxBytes - totalBytesRead);
|
|
122016
|
-
}
|
|
122017
|
-
totalBytesRead += chunk2.byteLength;
|
|
122018
|
-
yield range2.refCount > 1 ? chunk2.slice() : chunk2;
|
|
122019
|
-
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
122020
|
-
if (totalBytesRead >= maxBytes) {
|
|
122021
|
-
break rangeLoop;
|
|
122022
|
-
}
|
|
122023
|
-
}
|
|
122024
|
-
}
|
|
122025
|
-
rangeList.remove(term.range.start, term.range.end);
|
|
122026
|
-
continue;
|
|
122027
|
-
}
|
|
122299
|
+
return this.results;
|
|
122300
|
+
}
|
|
122301
|
+
streamSync() {
|
|
122302
|
+
if (this.path.isUnknown()) {
|
|
122303
|
+
this.path.lstatSync();
|
|
122028
122304
|
}
|
|
122029
|
-
|
|
122030
|
-
|
|
122031
|
-
|
|
122032
|
-
|
|
122033
|
-
|
|
122034
|
-
|
|
122035
|
-
|
|
122305
|
+
this.walkCBSync(this.path, this.patterns, () => this.results.end());
|
|
122306
|
+
return this.results;
|
|
122307
|
+
}
|
|
122308
|
+
}
|
|
122309
|
+
|
|
122310
|
+
// if no process global, just call it linux.
|
|
122311
|
+
// so we default to case-sensitive, / separators
|
|
122312
|
+
const defaultPlatform = (typeof process === 'object' &&
|
|
122313
|
+
process &&
|
|
122314
|
+
typeof process.platform === 'string') ?
|
|
122315
|
+
process.platform
|
|
122316
|
+
: 'linux';
|
|
122317
|
+
/**
|
|
122318
|
+
* An object that can perform glob pattern traversals.
|
|
122319
|
+
*/
|
|
122320
|
+
class Glob {
|
|
122321
|
+
absolute;
|
|
122322
|
+
cwd;
|
|
122323
|
+
root;
|
|
122324
|
+
dot;
|
|
122325
|
+
dotRelative;
|
|
122326
|
+
follow;
|
|
122327
|
+
ignore;
|
|
122328
|
+
magicalBraces;
|
|
122329
|
+
mark;
|
|
122330
|
+
matchBase;
|
|
122331
|
+
maxDepth;
|
|
122332
|
+
nobrace;
|
|
122333
|
+
nocase;
|
|
122334
|
+
nodir;
|
|
122335
|
+
noext;
|
|
122336
|
+
noglobstar;
|
|
122337
|
+
pattern;
|
|
122338
|
+
platform;
|
|
122339
|
+
realpath;
|
|
122340
|
+
scurry;
|
|
122341
|
+
stat;
|
|
122342
|
+
signal;
|
|
122343
|
+
windowsPathsNoEscape;
|
|
122344
|
+
withFileTypes;
|
|
122345
|
+
includeChildMatches;
|
|
122346
|
+
/**
|
|
122347
|
+
* The options provided to the constructor.
|
|
122348
|
+
*/
|
|
122349
|
+
opts;
|
|
122350
|
+
/**
|
|
122351
|
+
* An array of parsed immutable {@link Pattern} objects.
|
|
122352
|
+
*/
|
|
122353
|
+
patterns;
|
|
122354
|
+
/**
|
|
122355
|
+
* All options are stored as properties on the `Glob` object.
|
|
122356
|
+
*
|
|
122357
|
+
* See {@link GlobOptions} for full options descriptions.
|
|
122358
|
+
*
|
|
122359
|
+
* Note that a previous `Glob` object can be passed as the
|
|
122360
|
+
* `GlobOptions` to another `Glob` instantiation to re-use settings
|
|
122361
|
+
* and caches with a new pattern.
|
|
122362
|
+
*
|
|
122363
|
+
* Traversal functions can be called multiple times to run the walk
|
|
122364
|
+
* again.
|
|
122365
|
+
*/
|
|
122366
|
+
constructor(pattern, opts) {
|
|
122367
|
+
/* c8 ignore start */
|
|
122368
|
+
if (!opts)
|
|
122369
|
+
throw new TypeError('glob options required');
|
|
122370
|
+
/* c8 ignore stop */
|
|
122371
|
+
this.withFileTypes = !!opts.withFileTypes;
|
|
122372
|
+
this.signal = opts.signal;
|
|
122373
|
+
this.follow = !!opts.follow;
|
|
122374
|
+
this.dot = !!opts.dot;
|
|
122375
|
+
this.dotRelative = !!opts.dotRelative;
|
|
122376
|
+
this.nodir = !!opts.nodir;
|
|
122377
|
+
this.mark = !!opts.mark;
|
|
122378
|
+
if (!opts.cwd) {
|
|
122379
|
+
this.cwd = '';
|
|
122036
122380
|
}
|
|
122037
|
-
|
|
122038
|
-
|
|
122039
|
-
log("readBytesToSkip", readBytesToSkip);
|
|
122040
|
-
let resp = await customFetch(fetchInfo.url, {
|
|
122041
|
-
headers: {
|
|
122042
|
-
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
122043
|
-
}
|
|
122044
|
-
});
|
|
122045
|
-
if (resp.status === 403) {
|
|
122046
|
-
reconstructionInfo = await reloadReconstructionInfo();
|
|
122047
|
-
resp = await customFetch(fetchInfo.url, {
|
|
122048
|
-
headers: {
|
|
122049
|
-
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
122050
|
-
}
|
|
122051
|
-
});
|
|
122381
|
+
else if (opts.cwd instanceof URL || opts.cwd.startsWith('file://')) {
|
|
122382
|
+
opts.cwd = fileURLToPath(opts.cwd);
|
|
122052
122383
|
}
|
|
122053
|
-
|
|
122054
|
-
|
|
122384
|
+
this.cwd = opts.cwd || '';
|
|
122385
|
+
this.root = opts.root;
|
|
122386
|
+
this.magicalBraces = !!opts.magicalBraces;
|
|
122387
|
+
this.nobrace = !!opts.nobrace;
|
|
122388
|
+
this.noext = !!opts.noext;
|
|
122389
|
+
this.realpath = !!opts.realpath;
|
|
122390
|
+
this.absolute = opts.absolute;
|
|
122391
|
+
this.includeChildMatches = opts.includeChildMatches !== false;
|
|
122392
|
+
this.noglobstar = !!opts.noglobstar;
|
|
122393
|
+
this.matchBase = !!opts.matchBase;
|
|
122394
|
+
this.maxDepth =
|
|
122395
|
+
typeof opts.maxDepth === 'number' ? opts.maxDepth : Infinity;
|
|
122396
|
+
this.stat = !!opts.stat;
|
|
122397
|
+
this.ignore = opts.ignore;
|
|
122398
|
+
if (this.withFileTypes && this.absolute !== undefined) {
|
|
122399
|
+
throw new Error('cannot set absolute and withFileTypes:true');
|
|
122055
122400
|
}
|
|
122056
|
-
|
|
122057
|
-
|
|
122058
|
-
resp.headers.get("content-length"),
|
|
122059
|
-
"range",
|
|
122060
|
-
fetchInfo.url_range,
|
|
122061
|
-
resp.headers.get("content-range")
|
|
122062
|
-
);
|
|
122063
|
-
const reader = resp.body?.getReader();
|
|
122064
|
-
if (!reader) {
|
|
122065
|
-
throw new Error("Failed to get reader from response body");
|
|
122401
|
+
if (typeof pattern === 'string') {
|
|
122402
|
+
pattern = [pattern];
|
|
122066
122403
|
}
|
|
122067
|
-
|
|
122068
|
-
|
|
122069
|
-
|
|
122070
|
-
|
|
122071
|
-
|
|
122072
|
-
|
|
122073
|
-
|
|
122074
|
-
|
|
122075
|
-
|
|
122076
|
-
|
|
122077
|
-
log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip);
|
|
122078
|
-
if (!result.value) {
|
|
122079
|
-
log("no data in result, cancelled", result);
|
|
122080
|
-
continue;
|
|
122081
|
-
}
|
|
122082
|
-
totalFetchBytes += result.value.byteLength;
|
|
122083
|
-
if (leftoverBytes) {
|
|
122084
|
-
result.value = combineUint8Arrays(leftoverBytes, result.value);
|
|
122085
|
-
leftoverBytes = void 0;
|
|
122404
|
+
this.windowsPathsNoEscape =
|
|
122405
|
+
!!opts.windowsPathsNoEscape ||
|
|
122406
|
+
opts.allowWindowsEscape ===
|
|
122407
|
+
false;
|
|
122408
|
+
if (this.windowsPathsNoEscape) {
|
|
122409
|
+
pattern = pattern.map(p => p.replace(/\\/g, '/'));
|
|
122410
|
+
}
|
|
122411
|
+
if (this.matchBase) {
|
|
122412
|
+
if (opts.noglobstar) {
|
|
122413
|
+
throw new TypeError('base matching requires globstar');
|
|
122086
122414
|
}
|
|
122087
|
-
|
|
122088
|
-
|
|
122089
|
-
|
|
122090
|
-
|
|
122091
|
-
|
|
122092
|
-
|
|
122093
|
-
|
|
122094
|
-
|
|
122095
|
-
|
|
122096
|
-
|
|
122097
|
-
uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16
|
|
122098
|
-
};
|
|
122099
|
-
log("chunk header", chunkHeader, "to skip", readBytesToSkip);
|
|
122100
|
-
if (chunkHeader.version !== 0) {
|
|
122101
|
-
throw new Error(`Unsupported chunk version ${chunkHeader.version}`);
|
|
122102
|
-
}
|
|
122103
|
-
if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) {
|
|
122104
|
-
throw new Error(
|
|
122105
|
-
`Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}`
|
|
122106
|
-
);
|
|
122107
|
-
}
|
|
122108
|
-
if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) {
|
|
122109
|
-
leftoverBytes = result.value;
|
|
122110
|
-
continue fetchData;
|
|
122111
|
-
}
|
|
122112
|
-
result.value = result.value.slice(XET_CHUNK_HEADER_BYTES);
|
|
122113
|
-
let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes(
|
|
122114
|
-
decompress(
|
|
122115
|
-
result.value.slice(0, chunkHeader.compressed_length),
|
|
122116
|
-
chunkHeader.uncompressed_length
|
|
122117
|
-
)
|
|
122118
|
-
) : result.value.slice(0, chunkHeader.compressed_length);
|
|
122119
|
-
const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end);
|
|
122120
|
-
const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end;
|
|
122121
|
-
const minRefCountToStore = shouldYield ? 2 : 1;
|
|
122122
|
-
let stored = false;
|
|
122123
|
-
if (range2 && range2.refCount >= minRefCountToStore) {
|
|
122124
|
-
range2.data ??= [];
|
|
122125
|
-
range2.data.push(uncompressed);
|
|
122126
|
-
stored = true;
|
|
122127
|
-
}
|
|
122128
|
-
if (shouldYield) {
|
|
122129
|
-
if (readBytesToSkip) {
|
|
122130
|
-
const skipped = Math.min(readBytesToSkip, uncompressed.byteLength);
|
|
122131
|
-
uncompressed = uncompressed.slice(readBytesToSkip);
|
|
122132
|
-
readBytesToSkip -= skipped;
|
|
122133
|
-
}
|
|
122134
|
-
if (uncompressed.byteLength > maxBytes - totalBytesRead) {
|
|
122135
|
-
uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead);
|
|
122136
|
-
}
|
|
122137
|
-
if (uncompressed.byteLength) {
|
|
122138
|
-
log(
|
|
122139
|
-
"yield",
|
|
122140
|
-
uncompressed.byteLength,
|
|
122141
|
-
"bytes",
|
|
122142
|
-
result.value.byteLength,
|
|
122143
|
-
"total read",
|
|
122144
|
-
totalBytesRead,
|
|
122145
|
-
stored
|
|
122146
|
-
);
|
|
122147
|
-
totalBytesRead += uncompressed.byteLength;
|
|
122148
|
-
yield stored ? uncompressed.slice() : uncompressed;
|
|
122149
|
-
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
122150
|
-
}
|
|
122151
|
-
}
|
|
122152
|
-
chunkIndex++;
|
|
122153
|
-
result.value = result.value.slice(chunkHeader.compressed_length);
|
|
122415
|
+
pattern = pattern.map(p => (p.includes('/') ? p : `./**/${p}`));
|
|
122416
|
+
}
|
|
122417
|
+
this.pattern = pattern;
|
|
122418
|
+
this.platform = opts.platform || defaultPlatform;
|
|
122419
|
+
this.opts = { ...opts, platform: this.platform };
|
|
122420
|
+
if (opts.scurry) {
|
|
122421
|
+
this.scurry = opts.scurry;
|
|
122422
|
+
if (opts.nocase !== undefined &&
|
|
122423
|
+
opts.nocase !== opts.scurry.nocase) {
|
|
122424
|
+
throw new Error('nocase option contradicts provided scurry option');
|
|
122154
122425
|
}
|
|
122155
|
-
}
|
|
122156
|
-
if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) {
|
|
122157
|
-
log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes);
|
|
122158
|
-
log("failed to fetch all data for term", term.hash);
|
|
122159
|
-
throw new Error(
|
|
122160
|
-
`Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}`
|
|
122161
|
-
);
|
|
122162
122426
|
}
|
|
122163
|
-
|
|
122164
|
-
|
|
122165
|
-
|
|
122166
|
-
|
|
122427
|
+
else {
|
|
122428
|
+
const Scurry = opts.platform === 'win32' ? PathScurryWin32
|
|
122429
|
+
: opts.platform === 'darwin' ? PathScurryDarwin
|
|
122430
|
+
: opts.platform ? PathScurryPosix
|
|
122431
|
+
: PathScurry;
|
|
122432
|
+
this.scurry = new Scurry(this.cwd, {
|
|
122433
|
+
nocase: opts.nocase,
|
|
122434
|
+
fs: opts.fs,
|
|
122435
|
+
});
|
|
122436
|
+
}
|
|
122437
|
+
this.nocase = this.scurry.nocase;
|
|
122438
|
+
// If you do nocase:true on a case-sensitive file system, then
|
|
122439
|
+
// we need to use regexps instead of strings for non-magic
|
|
122440
|
+
// path portions, because statting `aBc` won't return results
|
|
122441
|
+
// for the file `AbC` for example.
|
|
122442
|
+
const nocaseMagicOnly = this.platform === 'darwin' || this.platform === 'win32';
|
|
122443
|
+
const mmo = {
|
|
122444
|
+
// default nocase based on platform
|
|
122445
|
+
...opts,
|
|
122446
|
+
dot: this.dot,
|
|
122447
|
+
matchBase: this.matchBase,
|
|
122448
|
+
nobrace: this.nobrace,
|
|
122449
|
+
nocase: this.nocase,
|
|
122450
|
+
nocaseMagicOnly,
|
|
122451
|
+
nocomment: true,
|
|
122452
|
+
noext: this.noext,
|
|
122453
|
+
nonegate: true,
|
|
122454
|
+
optimizationLevel: 2,
|
|
122455
|
+
platform: this.platform,
|
|
122456
|
+
windowsPathsNoEscape: this.windowsPathsNoEscape,
|
|
122457
|
+
debug: !!this.opts.debug,
|
|
122458
|
+
};
|
|
122459
|
+
const mms = this.pattern.map(p => new Minimatch(p, mmo));
|
|
122460
|
+
const [matchSet, globParts] = mms.reduce((set, m) => {
|
|
122461
|
+
set[0].push(...m.set);
|
|
122462
|
+
set[1].push(...m.globParts);
|
|
122463
|
+
return set;
|
|
122464
|
+
}, [[], []]);
|
|
122465
|
+
this.patterns = matchSet.map((set, i) => {
|
|
122466
|
+
const g = globParts[i];
|
|
122467
|
+
/* c8 ignore start */
|
|
122468
|
+
if (!g)
|
|
122469
|
+
throw new Error('invalid pattern object');
|
|
122470
|
+
/* c8 ignore stop */
|
|
122471
|
+
return new Pattern(set, g, 0, this.platform);
|
|
122472
|
+
});
|
|
122167
122473
|
}
|
|
122168
|
-
|
|
122169
|
-
|
|
122170
|
-
|
|
122171
|
-
|
|
122172
|
-
|
|
122173
|
-
|
|
122174
|
-
|
|
122175
|
-
|
|
122176
|
-
|
|
122177
|
-
|
|
122178
|
-
|
|
122179
|
-
|
|
122180
|
-
|
|
122181
|
-
|
|
122182
|
-
|
|
122183
|
-
|
|
122184
|
-
}
|
|
122185
|
-
},
|
|
122186
|
-
type: "bytes"
|
|
122187
|
-
// todo: when Safari supports it, add autoAllocateChunkSize param
|
|
122188
|
-
},
|
|
122189
|
-
// todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function
|
|
122190
|
-
{
|
|
122191
|
-
highWaterMark: 1e3
|
|
122192
|
-
// 1_000 chunks for ~1MB of RAM
|
|
122193
|
-
}
|
|
122194
|
-
);
|
|
122195
|
-
}
|
|
122196
|
-
async arrayBuffer() {
|
|
122197
|
-
const result = await this.#fetch();
|
|
122198
|
-
return new Response(result).arrayBuffer();
|
|
122199
|
-
}
|
|
122200
|
-
async text() {
|
|
122201
|
-
const result = await this.#fetch();
|
|
122202
|
-
return new Response(result).text();
|
|
122203
|
-
}
|
|
122204
|
-
async response() {
|
|
122205
|
-
const result = await this.#fetch();
|
|
122206
|
-
return new Response(result);
|
|
122207
|
-
}
|
|
122208
|
-
stream() {
|
|
122209
|
-
const stream = new TransformStream();
|
|
122210
|
-
this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
122211
|
-
return stream.readable;
|
|
122212
|
-
}
|
|
122213
|
-
};
|
|
122214
|
-
var jwtPromises = /* @__PURE__ */ new Map();
|
|
122215
|
-
var jwts = /* @__PURE__ */ new Map();
|
|
122216
|
-
function cacheKey(params) {
|
|
122217
|
-
return JSON.stringify([params.refreshUrl, params.initialAccessToken]);
|
|
122218
|
-
}
|
|
122219
|
-
function bg4_regroup_bytes(bytes) {
|
|
122220
|
-
const split = Math.floor(bytes.byteLength / 4);
|
|
122221
|
-
const rem = bytes.byteLength % 4;
|
|
122222
|
-
const g1_pos = split + (rem >= 1 ? 1 : 0);
|
|
122223
|
-
const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0);
|
|
122224
|
-
const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0);
|
|
122225
|
-
const ret = new Uint8Array(bytes.byteLength);
|
|
122226
|
-
for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) {
|
|
122227
|
-
ret[i] = bytes[j];
|
|
122228
|
-
}
|
|
122229
|
-
for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) {
|
|
122230
|
-
ret[i] = bytes[j];
|
|
122231
|
-
}
|
|
122232
|
-
for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) {
|
|
122233
|
-
ret[i] = bytes[j];
|
|
122234
|
-
}
|
|
122235
|
-
for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) {
|
|
122236
|
-
ret[i] = bytes[j];
|
|
122237
|
-
}
|
|
122238
|
-
return ret;
|
|
122239
|
-
}
|
|
122240
|
-
async function getAccessToken(initialAccessToken, customFetch, refreshUrl) {
|
|
122241
|
-
const key = cacheKey({ refreshUrl, initialAccessToken });
|
|
122242
|
-
const jwt = jwts.get(key);
|
|
122243
|
-
if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) {
|
|
122244
|
-
return { accessToken: jwt.accessToken, casUrl: jwt.casUrl };
|
|
122245
|
-
}
|
|
122246
|
-
const existingPromise = jwtPromises.get(key);
|
|
122247
|
-
if (existingPromise) {
|
|
122248
|
-
return existingPromise;
|
|
122249
|
-
}
|
|
122250
|
-
const promise = (async () => {
|
|
122251
|
-
const resp = await customFetch(refreshUrl, {
|
|
122252
|
-
headers: {
|
|
122253
|
-
...initialAccessToken ? {
|
|
122254
|
-
Authorization: `Bearer ${initialAccessToken}`
|
|
122255
|
-
} : {}
|
|
122256
|
-
}
|
|
122257
|
-
});
|
|
122258
|
-
if (!resp.ok) {
|
|
122259
|
-
throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`);
|
|
122474
|
+
async walk() {
|
|
122475
|
+
// Walkers always return array of Path objects, so we just have to
|
|
122476
|
+
// coerce them into the right shape. It will have already called
|
|
122477
|
+
// realpath() if the option was set to do so, so we know that's cached.
|
|
122478
|
+
// start out knowing the cwd, at least
|
|
122479
|
+
return [
|
|
122480
|
+
...(await new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
122481
|
+
...this.opts,
|
|
122482
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122483
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122484
|
+
: Infinity,
|
|
122485
|
+
platform: this.platform,
|
|
122486
|
+
nocase: this.nocase,
|
|
122487
|
+
includeChildMatches: this.includeChildMatches,
|
|
122488
|
+
}).walk()),
|
|
122489
|
+
];
|
|
122260
122490
|
}
|
|
122261
|
-
|
|
122262
|
-
|
|
122263
|
-
|
|
122264
|
-
|
|
122265
|
-
|
|
122266
|
-
|
|
122267
|
-
|
|
122268
|
-
|
|
122269
|
-
|
|
122270
|
-
|
|
122271
|
-
|
|
122272
|
-
|
|
122273
|
-
}
|
|
122491
|
+
walkSync() {
|
|
122492
|
+
return [
|
|
122493
|
+
...new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
122494
|
+
...this.opts,
|
|
122495
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122496
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122497
|
+
: Infinity,
|
|
122498
|
+
platform: this.platform,
|
|
122499
|
+
nocase: this.nocase,
|
|
122500
|
+
includeChildMatches: this.includeChildMatches,
|
|
122501
|
+
}).walkSync(),
|
|
122502
|
+
];
|
|
122274
122503
|
}
|
|
122275
|
-
|
|
122276
|
-
|
|
122277
|
-
|
|
122278
|
-
|
|
122279
|
-
|
|
122504
|
+
stream() {
|
|
122505
|
+
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
122506
|
+
...this.opts,
|
|
122507
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122508
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122509
|
+
: Infinity,
|
|
122510
|
+
platform: this.platform,
|
|
122511
|
+
nocase: this.nocase,
|
|
122512
|
+
includeChildMatches: this.includeChildMatches,
|
|
122513
|
+
}).stream();
|
|
122514
|
+
}
|
|
122515
|
+
streamSync() {
|
|
122516
|
+
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
122517
|
+
...this.opts,
|
|
122518
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122519
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122520
|
+
: Infinity,
|
|
122521
|
+
platform: this.platform,
|
|
122522
|
+
nocase: this.nocase,
|
|
122523
|
+
includeChildMatches: this.includeChildMatches,
|
|
122524
|
+
}).streamSync();
|
|
122525
|
+
}
|
|
122526
|
+
/**
|
|
122527
|
+
* Default sync iteration function. Returns a Generator that
|
|
122528
|
+
* iterates over the results.
|
|
122529
|
+
*/
|
|
122530
|
+
iterateSync() {
|
|
122531
|
+
return this.streamSync()[Symbol.iterator]();
|
|
122532
|
+
}
|
|
122533
|
+
[Symbol.iterator]() {
|
|
122534
|
+
return this.iterateSync();
|
|
122535
|
+
}
|
|
122536
|
+
/**
|
|
122537
|
+
* Default async iteration function. Returns an AsyncGenerator that
|
|
122538
|
+
* iterates over the results.
|
|
122539
|
+
*/
|
|
122540
|
+
iterate() {
|
|
122541
|
+
return this.stream()[Symbol.asyncIterator]();
|
|
122542
|
+
}
|
|
122543
|
+
[Symbol.asyncIterator]() {
|
|
122544
|
+
return this.iterate();
|
|
122280
122545
|
}
|
|
122281
|
-
jwts.set(key, jwt2);
|
|
122282
|
-
return {
|
|
122283
|
-
accessToken: json.accessToken,
|
|
122284
|
-
casUrl: json.casUrl
|
|
122285
|
-
};
|
|
122286
|
-
})();
|
|
122287
|
-
jwtPromises.set(key, promise);
|
|
122288
|
-
return promise;
|
|
122289
122546
|
}
|
|
122290
122547
|
|
|
122291
|
-
|
|
122292
|
-
|
|
122293
|
-
|
|
122294
|
-
|
|
122295
|
-
|
|
122296
|
-
|
|
122297
|
-
|
|
122298
|
-
|
|
122299
|
-
|
|
122300
|
-
|
|
122301
|
-
|
|
122302
|
-
|
|
122303
|
-
|
|
122304
|
-
|
|
122305
|
-
const supportRange = response.headers.get("accept-ranges") === "bytes";
|
|
122306
|
-
if (!supportRange || size < (opts?.cacheBelow ?? 1e6)) {
|
|
122307
|
-
return await (await customFetch(url)).blob();
|
|
122548
|
+
/**
|
|
122549
|
+
* Return true if the patterns provided contain any magic glob characters,
|
|
122550
|
+
* given the options provided.
|
|
122551
|
+
*
|
|
122552
|
+
* Brace expansion is not considered "magic" unless the `magicalBraces` option
|
|
122553
|
+
* is set, as brace expansion just turns one string into an array of strings.
|
|
122554
|
+
* So a pattern like `'x{a,b}y'` would return `false`, because `'xay'` and
|
|
122555
|
+
* `'xby'` both do not contain any magic glob characters, and it's treated the
|
|
122556
|
+
* same as if you had called it on `['xay', 'xby']`. When `magicalBraces:true`
|
|
122557
|
+
* is in the options, brace expansion _is_ treated as a pattern having magic.
|
|
122558
|
+
*/
|
|
122559
|
+
const hasMagic = (pattern, options = {}) => {
|
|
122560
|
+
if (!Array.isArray(pattern)) {
|
|
122561
|
+
pattern = [pattern];
|
|
122308
122562
|
}
|
|
122309
|
-
|
|
122310
|
-
|
|
122311
|
-
|
|
122312
|
-
start;
|
|
122313
|
-
end;
|
|
122314
|
-
contentType;
|
|
122315
|
-
full;
|
|
122316
|
-
fetch;
|
|
122317
|
-
accessToken;
|
|
122318
|
-
constructor(url, start, end, contentType, full, customFetch, accessToken) {
|
|
122319
|
-
super([]);
|
|
122320
|
-
this.url = url;
|
|
122321
|
-
this.start = start;
|
|
122322
|
-
this.end = end;
|
|
122323
|
-
this.contentType = contentType;
|
|
122324
|
-
this.full = full;
|
|
122325
|
-
this.fetch = customFetch;
|
|
122326
|
-
this.accessToken = accessToken;
|
|
122327
|
-
}
|
|
122328
|
-
get size() {
|
|
122329
|
-
return this.end - this.start;
|
|
122330
|
-
}
|
|
122331
|
-
get type() {
|
|
122332
|
-
return this.contentType;
|
|
122333
|
-
}
|
|
122334
|
-
slice(start = 0, end = this.size) {
|
|
122335
|
-
const slice = new WebBlob(
|
|
122336
|
-
this.url,
|
|
122337
|
-
this.start + start,
|
|
122338
|
-
Math.min(this.start + end, this.end),
|
|
122339
|
-
this.contentType,
|
|
122340
|
-
start === 0 && end === this.size ? this.full : false,
|
|
122341
|
-
this.fetch,
|
|
122342
|
-
this.accessToken
|
|
122343
|
-
);
|
|
122344
|
-
return slice;
|
|
122345
|
-
}
|
|
122346
|
-
async arrayBuffer() {
|
|
122347
|
-
const result = await this.fetchRange();
|
|
122348
|
-
return result.arrayBuffer();
|
|
122349
|
-
}
|
|
122350
|
-
async text() {
|
|
122351
|
-
const result = await this.fetchRange();
|
|
122352
|
-
return result.text();
|
|
122353
|
-
}
|
|
122354
|
-
stream() {
|
|
122355
|
-
const stream = new TransformStream();
|
|
122356
|
-
this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
122357
|
-
return stream.readable;
|
|
122358
|
-
}
|
|
122359
|
-
fetchRange() {
|
|
122360
|
-
const fetch2 = this.fetch;
|
|
122361
|
-
if (this.full) {
|
|
122362
|
-
return fetch2(this.url, {
|
|
122363
|
-
...this.accessToken && {
|
|
122364
|
-
headers: {
|
|
122365
|
-
Authorization: `Bearer ${this.accessToken}`
|
|
122366
|
-
}
|
|
122367
|
-
}
|
|
122368
|
-
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
122563
|
+
for (const p of pattern) {
|
|
122564
|
+
if (new Minimatch(p, options).hasMagic())
|
|
122565
|
+
return true;
|
|
122369
122566
|
}
|
|
122370
|
-
return
|
|
122371
|
-
headers: {
|
|
122372
|
-
Range: `bytes=${this.start}-${this.end - 1}`,
|
|
122373
|
-
...this.accessToken && { Authorization: `Bearer ${this.accessToken}` }
|
|
122374
|
-
}
|
|
122375
|
-
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
122376
|
-
}
|
|
122567
|
+
return false;
|
|
122377
122568
|
};
|
|
122378
122569
|
|
|
122379
|
-
|
|
122380
|
-
|
|
122381
|
-
|
|
122382
|
-
|
|
122570
|
+
function globStreamSync(pattern, options = {}) {
|
|
122571
|
+
return new Glob(pattern, options).streamSync();
|
|
122572
|
+
}
|
|
122573
|
+
function globStream(pattern, options = {}) {
|
|
122574
|
+
return new Glob(pattern, options).stream();
|
|
122575
|
+
}
|
|
122576
|
+
function globSync(pattern, options = {}) {
|
|
122577
|
+
return new Glob(pattern, options).walkSync();
|
|
122578
|
+
}
|
|
122579
|
+
async function glob_(pattern, options = {}) {
|
|
122580
|
+
return new Glob(pattern, options).walk();
|
|
122581
|
+
}
|
|
122582
|
+
function globIterateSync(pattern, options = {}) {
|
|
122583
|
+
return new Glob(pattern, options).iterateSync();
|
|
122584
|
+
}
|
|
122585
|
+
function globIterate(pattern, options = {}) {
|
|
122586
|
+
return new Glob(pattern, options).iterate();
|
|
122383
122587
|
}
|
|
122588
|
+
// aliases: glob.sync.stream() glob.stream.sync() glob.sync() etc
|
|
122589
|
+
const streamSync = globStreamSync;
|
|
122590
|
+
const stream = Object.assign(globStream, { sync: globStreamSync });
|
|
122591
|
+
const iterateSync = globIterateSync;
|
|
122592
|
+
const iterate = Object.assign(globIterate, {
|
|
122593
|
+
sync: globIterateSync,
|
|
122594
|
+
});
|
|
122595
|
+
const sync = Object.assign(globSync, {
|
|
122596
|
+
stream: globStreamSync,
|
|
122597
|
+
iterate: globIterateSync,
|
|
122598
|
+
});
|
|
122599
|
+
const glob = Object.assign(glob_, {
|
|
122600
|
+
glob: glob_,
|
|
122601
|
+
globSync,
|
|
122602
|
+
sync,
|
|
122603
|
+
globStream,
|
|
122604
|
+
stream,
|
|
122605
|
+
globStreamSync,
|
|
122606
|
+
streamSync,
|
|
122607
|
+
globIterate,
|
|
122608
|
+
iterate,
|
|
122609
|
+
globIterateSync,
|
|
122610
|
+
iterateSync,
|
|
122611
|
+
Glob,
|
|
122612
|
+
hasMagic,
|
|
122613
|
+
escape: escape$1,
|
|
122614
|
+
unescape: unescape$1,
|
|
122615
|
+
});
|
|
122616
|
+
glob.glob = glob;
|
|
122384
122617
|
|
|
122385
|
-
|
|
122386
|
-
|
|
122387
|
-
|
|
122388
|
-
const repoId = toRepoId(params.repo);
|
|
122389
|
-
const hubUrl = params.hubUrl ?? HUB_URL;
|
|
122390
|
-
const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}/${encodeURIComponent(params.revision ?? "main")}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : "");
|
|
122391
|
-
const resp = await (params.fetch ?? fetch)(url, {
|
|
122392
|
-
method: "GET",
|
|
122393
|
-
headers: {
|
|
122394
|
-
...accessToken && {
|
|
122395
|
-
Authorization: `Bearer ${accessToken}`
|
|
122396
|
-
},
|
|
122397
|
-
Range: "bytes=0-0",
|
|
122398
|
-
Accept: "application/vnd.xet-fileinfo+json, */*"
|
|
122618
|
+
function matchesQuantizationVariant({ filePath, variant }) {
|
|
122619
|
+
if (!variant) {
|
|
122620
|
+
return false;
|
|
122399
122621
|
}
|
|
122400
|
-
|
|
122401
|
-
|
|
122402
|
-
|
|
122403
|
-
|
|
122404
|
-
|
|
122405
|
-
|
|
122406
|
-
}
|
|
122407
|
-
let size;
|
|
122408
|
-
let xetInfo;
|
|
122409
|
-
if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) {
|
|
122410
|
-
size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid");
|
|
122411
|
-
if (isNaN(size)) {
|
|
122412
|
-
throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header");
|
|
122622
|
+
const escapedVariant = variant.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
122623
|
+
const matcher = new RegExp(`(^|[\\-./_])${escapedVariant}(?=[\\-./]|$)`, "i");
|
|
122624
|
+
const normalizedPath = filePath.replace(/\\/g, "/");
|
|
122625
|
+
const segments = normalizedPath.split("/").filter(Boolean);
|
|
122626
|
+
if (segments.length === 0) {
|
|
122627
|
+
return false;
|
|
122413
122628
|
}
|
|
122414
|
-
const
|
|
122415
|
-
|
|
122416
|
-
|
|
122417
|
-
try {
|
|
122418
|
-
return new URL(links["xet-reconstruction-info"]);
|
|
122419
|
-
} catch {
|
|
122420
|
-
return null;
|
|
122421
|
-
}
|
|
122422
|
-
})();
|
|
122423
|
-
const refreshUrl = (() => {
|
|
122424
|
-
try {
|
|
122425
|
-
return new URL(links["xet-auth"]);
|
|
122426
|
-
} catch {
|
|
122427
|
-
return null;
|
|
122428
|
-
}
|
|
122429
|
-
})();
|
|
122430
|
-
if (!hash2) {
|
|
122431
|
-
throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header");
|
|
122629
|
+
const filename = segments[segments.length - 1].replace(/\.gguf$/i, "");
|
|
122630
|
+
if (matcher.test(filename)) {
|
|
122631
|
+
return true;
|
|
122432
122632
|
}
|
|
122433
|
-
|
|
122434
|
-
|
|
122633
|
+
return segments.slice(0, -1).some(segment => matcher.test(segment));
|
|
122634
|
+
}
|
|
122635
|
+
async function findQuantizedModelTarget({ model, path }) {
|
|
122636
|
+
if (model.source.type === "storage") {
|
|
122637
|
+
throw new Error("Model storage not supported yet");
|
|
122435
122638
|
}
|
|
122436
|
-
|
|
122437
|
-
|
|
122438
|
-
refreshUrl,
|
|
122439
|
-
reconstructionUrl
|
|
122440
|
-
};
|
|
122441
|
-
}
|
|
122442
|
-
if (size === void 0 || isNaN(size)) {
|
|
122443
|
-
const contentRangeHeader = resp.headers.get("content-range");
|
|
122444
|
-
if (!contentRangeHeader) {
|
|
122445
|
-
throw new InvalidApiResponseFormatError("Expected size information");
|
|
122639
|
+
if (model.format !== "gguf") {
|
|
122640
|
+
throw new Error(`Model format not supported: ${model.format}`);
|
|
122446
122641
|
}
|
|
122447
|
-
const [,
|
|
122448
|
-
|
|
122449
|
-
|
|
122450
|
-
|
|
122642
|
+
const [, variant = null] = model.source.slug.split(":");
|
|
122643
|
+
const modelFiles = (await glob("**/*.gguf", {
|
|
122644
|
+
absolute: true,
|
|
122645
|
+
cwd: path,
|
|
122646
|
+
nodir: true
|
|
122647
|
+
})).filter(file => !/(mmproj|clip)/i.test(file));
|
|
122648
|
+
if (modelFiles.length <= 0) {
|
|
122649
|
+
throw new Error(`No models found for format: ${model.format}`);
|
|
122451
122650
|
}
|
|
122452
|
-
|
|
122453
|
-
|
|
122454
|
-
|
|
122455
|
-
|
|
122456
|
-
|
|
122457
|
-
|
|
122458
|
-
|
|
122459
|
-
|
|
122460
|
-
|
|
122461
|
-
// Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it.
|
|
122462
|
-
url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url
|
|
122463
|
-
};
|
|
122464
|
-
}
|
|
122465
|
-
|
|
122466
|
-
// src/lib/download-file.ts
|
|
122467
|
-
async function downloadFile(params) {
|
|
122468
|
-
const accessToken = checkCredentials(params);
|
|
122469
|
-
const info = params.downloadInfo ?? await fileDownloadInfo({
|
|
122470
|
-
accessToken,
|
|
122471
|
-
repo: params.repo,
|
|
122472
|
-
path: params.path,
|
|
122473
|
-
revision: params.revision,
|
|
122474
|
-
hubUrl: params.hubUrl,
|
|
122475
|
-
fetch: params.fetch,
|
|
122476
|
-
raw: params.raw
|
|
122477
|
-
});
|
|
122478
|
-
if (!info) {
|
|
122479
|
-
return null;
|
|
122480
|
-
}
|
|
122481
|
-
if (info.xet && params.xet) {
|
|
122482
|
-
return new XetBlob({
|
|
122483
|
-
refreshUrl: info.xet.refreshUrl.href,
|
|
122484
|
-
reconstructionUrl: info.xet.reconstructionUrl.href,
|
|
122485
|
-
fetch: params.fetch,
|
|
122486
|
-
accessToken,
|
|
122487
|
-
size: info.size
|
|
122488
|
-
});
|
|
122489
|
-
}
|
|
122490
|
-
return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken);
|
|
122651
|
+
modelFiles.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: "base" }));
|
|
122652
|
+
if (!variant) {
|
|
122653
|
+
return modelFiles[0];
|
|
122654
|
+
}
|
|
122655
|
+
const matches = modelFiles.filter(fileName => matchesQuantizationVariant({ filePath: fileName, variant: variant ?? "" }));
|
|
122656
|
+
if (matches.length === 0) {
|
|
122657
|
+
throw new Error(`No model found for format and variant: ${model.format} / ${variant}`);
|
|
122658
|
+
}
|
|
122659
|
+
return matches[0];
|
|
122491
122660
|
}
|
|
122492
122661
|
|
|
122493
|
-
|
|
122494
|
-
|
|
122495
|
-
|
|
122496
|
-
|
|
122497
|
-
|
|
122498
|
-
|
|
122499
|
-
|
|
122500
|
-
|
|
122501
|
-
accept: "application/json",
|
|
122502
|
-
...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0
|
|
122503
|
-
}
|
|
122504
|
-
});
|
|
122505
|
-
if (!res.ok) {
|
|
122506
|
-
throw await createApiError(res);
|
|
122662
|
+
const VLLM_START_ARGS = ["-m", "vllm.entrypoints.openai.api_server", "--host", "0.0.0.0"];
|
|
122663
|
+
const VLLM_EXECUTABLE = "python3";
|
|
122664
|
+
const DEFAULT_VLLM_CONTEXT_LENGTH = 2048;
|
|
122665
|
+
async function startVLLM({ enginePort, targetDirectory }) {
|
|
122666
|
+
const contextLength = Math.max(1, this.contextLength ?? DEFAULT_VLLM_CONTEXT_LENGTH);
|
|
122667
|
+
let modelPath = targetDirectory;
|
|
122668
|
+
if (this.model.format === "gguf") {
|
|
122669
|
+
modelPath = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
|
|
122507
122670
|
}
|
|
122508
|
-
const
|
|
122509
|
-
|
|
122510
|
-
|
|
122671
|
+
const engineConfig = this.engineConfig;
|
|
122672
|
+
const device = typeof engineConfig?.device === "string" ? engineConfig.device : process.env.VLLM_DEVICE;
|
|
122673
|
+
const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : process.env.VLLM_DTYPE;
|
|
122674
|
+
const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
|
|
122675
|
+
const args = [
|
|
122676
|
+
...VLLM_START_ARGS,
|
|
122677
|
+
"--port",
|
|
122678
|
+
String(enginePort),
|
|
122679
|
+
"--model",
|
|
122680
|
+
modelPath,
|
|
122681
|
+
"--served-model-name",
|
|
122682
|
+
SERVED_MODEL_NAME,
|
|
122683
|
+
"--max-model-len",
|
|
122684
|
+
String(contextLength),
|
|
122685
|
+
"--tensor-parallel-size",
|
|
122686
|
+
String(tensorParallelSize)
|
|
122687
|
+
];
|
|
122688
|
+
if (this.model.taskType === "embeddings") {
|
|
122689
|
+
args.push("--task", "embed");
|
|
122511
122690
|
}
|
|
122512
|
-
|
|
122513
|
-
|
|
122514
|
-
|
|
122691
|
+
args.push(...(await getChatTemplateEngineArgs({
|
|
122692
|
+
engine: this.engine,
|
|
122693
|
+
model: this.model,
|
|
122694
|
+
targetDirectory
|
|
122695
|
+
})));
|
|
122696
|
+
if (device) {
|
|
122697
|
+
args.push("--device", device);
|
|
122698
|
+
}
|
|
122699
|
+
if (dtype) {
|
|
122700
|
+
args.push("--dtype", dtype);
|
|
122701
|
+
}
|
|
122702
|
+
args.push(...parseExtraArgs(engineConfig?.extraArgs));
|
|
122703
|
+
if (this.model.multimodalEnabled) {
|
|
122704
|
+
args.push("--limit-mm-per-prompt", process.env.VLLM_MM_LIMIT ?? '{"image":5}');
|
|
122705
|
+
}
|
|
122706
|
+
if (process.env.VLLM_TRUST_REMOTE_CODE === "true") {
|
|
122707
|
+
args.push("--trust-remote-code");
|
|
122708
|
+
}
|
|
122709
|
+
return createEngineProcess({ args, bin: VLLM_EXECUTABLE, logger: this.logger });
|
|
122515
122710
|
}
|
|
122516
122711
|
|
|
122517
122712
|
const ModelDownloadProgressSchema = object$1({
|
|
@@ -123114,6 +123309,11 @@ async function startLlamacpp({ enginePort, targetDirectory }) {
|
|
|
123114
123309
|
"--ctx-size",
|
|
123115
123310
|
String(contextLength)
|
|
123116
123311
|
];
|
|
123312
|
+
args.push(...(await getChatTemplateEngineArgs({
|
|
123313
|
+
engine: this.engine,
|
|
123314
|
+
model: this.model,
|
|
123315
|
+
targetDirectory
|
|
123316
|
+
})));
|
|
123117
123317
|
if (this.model.taskType === "embeddings") {
|
|
123118
123318
|
args.push("--embedding");
|
|
123119
123319
|
}
|
|
@@ -123253,6 +123453,11 @@ async function startSGLang({ enginePort, targetDirectory }) {
|
|
|
123253
123453
|
if (this.model.taskType === "embeddings") {
|
|
123254
123454
|
args.push("--task", "embed");
|
|
123255
123455
|
}
|
|
123456
|
+
args.push(...(await getChatTemplateEngineArgs({
|
|
123457
|
+
engine: this.engine,
|
|
123458
|
+
model: this.model,
|
|
123459
|
+
targetDirectory
|
|
123460
|
+
})));
|
|
123256
123461
|
if (device) {
|
|
123257
123462
|
args.push("--device", device);
|
|
123258
123463
|
}
|
|
@@ -123333,6 +123538,15 @@ class ModelManager extends EventEmitter {
|
|
|
123333
123538
|
this.uniqueName = createModelStorageKey(this.model);
|
|
123334
123539
|
this.modelsDirectory = join(root, "models");
|
|
123335
123540
|
}
|
|
123541
|
+
async verifyChatTemplate() {
|
|
123542
|
+
return verifyEngineChatTemplate({
|
|
123543
|
+
engine: this.engine,
|
|
123544
|
+
enginePort: this.enginePort,
|
|
123545
|
+
logger: this.logger,
|
|
123546
|
+
model: this.model,
|
|
123547
|
+
targetDirectory: join(this.modelsDirectory, this.uniqueName)
|
|
123548
|
+
});
|
|
123549
|
+
}
|
|
123336
123550
|
async fetchOpenAI(path, opts) {
|
|
123337
123551
|
switch (this.engine) {
|
|
123338
123552
|
case "exllamav3":
|
|
@@ -123410,6 +123624,12 @@ class ModelManager extends EventEmitter {
|
|
|
123410
123624
|
finally {
|
|
123411
123625
|
await this.releaseDownloadLock();
|
|
123412
123626
|
}
|
|
123627
|
+
await materializeChatTemplate({
|
|
123628
|
+
engine: this.engine,
|
|
123629
|
+
huggingFaceToken: this.model.source.modelSecret,
|
|
123630
|
+
model: this.model,
|
|
123631
|
+
targetDirectory: join(this.modelsDirectory, this.uniqueName)
|
|
123632
|
+
});
|
|
123413
123633
|
break;
|
|
123414
123634
|
default: {
|
|
123415
123635
|
const engineType = this.engine;
|
|
@@ -124393,7 +124613,50 @@ function stripImagesFromBody(body) {
|
|
|
124393
124613
|
function isPlainObject$3(value) {
|
|
124394
124614
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
124395
124615
|
}
|
|
124396
|
-
|
|
124616
|
+
/**
|
|
124617
|
+
* Builds `chat_template_kwargs` for engines that read template variables at
|
|
124618
|
+
* render time. Only active when the model carries a template override or
|
|
124619
|
+
* thinking config; otherwise the body forwards untouched (engines like vLLM
|
|
124620
|
+
* consume top-level `reasoning_effort` natively for supported models).
|
|
124621
|
+
*
|
|
124622
|
+
* Precedence (highest wins):
|
|
124623
|
+
* 1. Request `chat_template_kwargs` (per key)
|
|
124624
|
+
* 2. Request top-level `reasoning_effort`
|
|
124625
|
+
* 3. Model thinking config defaults
|
|
124626
|
+
*/
|
|
124627
|
+
function applyChatTemplateKwargs({ body, model }) {
|
|
124628
|
+
const hasTemplateOverride = Boolean(model.chatTemplate);
|
|
124629
|
+
const thinkingConfig = model.thinkingConfig ?? null;
|
|
124630
|
+
if (!hasTemplateOverride && !thinkingConfig)
|
|
124631
|
+
return body;
|
|
124632
|
+
const requestKwargs = isPlainObject$3(body.chat_template_kwargs)
|
|
124633
|
+
? body.chat_template_kwargs
|
|
124634
|
+
: null;
|
|
124635
|
+
const requestEffort = typeof body.reasoning_effort === "string" ? body.reasoning_effort : null;
|
|
124636
|
+
const mergedKwargs = {};
|
|
124637
|
+
if (thinkingConfig?.enabled === false) {
|
|
124638
|
+
mergedKwargs.enable_thinking = false;
|
|
124639
|
+
}
|
|
124640
|
+
if (thinkingConfig?.effort) {
|
|
124641
|
+
mergedKwargs.reasoning_effort = thinkingConfig.effort;
|
|
124642
|
+
}
|
|
124643
|
+
if (requestKwargs) {
|
|
124644
|
+
Object.assign(mergedKwargs, requestKwargs);
|
|
124645
|
+
}
|
|
124646
|
+
const hasRequestReasoningEffort = requestKwargs !== null && "reasoning_effort" in requestKwargs;
|
|
124647
|
+
if (requestEffort && !hasRequestReasoningEffort) {
|
|
124648
|
+
mergedKwargs.reasoning_effort = requestEffort;
|
|
124649
|
+
}
|
|
124650
|
+
const payload = { ...body };
|
|
124651
|
+
if (Object.keys(mergedKwargs).length > 0) {
|
|
124652
|
+
payload.chat_template_kwargs = mergedKwargs;
|
|
124653
|
+
}
|
|
124654
|
+
if (requestEffort !== null) {
|
|
124655
|
+
delete payload.reasoning_effort;
|
|
124656
|
+
}
|
|
124657
|
+
return payload;
|
|
124658
|
+
}
|
|
124659
|
+
function serializeRequestBody$1(body, { model, path } = {}) {
|
|
124397
124660
|
if (!isPlainObject$3(body)) {
|
|
124398
124661
|
const payload = typeof body === "string" ? body : JSON.stringify(body);
|
|
124399
124662
|
return {
|
|
@@ -124401,7 +124664,10 @@ function serializeRequestBody$1(body) {
|
|
|
124401
124664
|
payload
|
|
124402
124665
|
};
|
|
124403
124666
|
}
|
|
124404
|
-
|
|
124667
|
+
let requestPayload = { ...body };
|
|
124668
|
+
if (path === "/v1/chat/completions" && model) {
|
|
124669
|
+
requestPayload = applyChatTemplateKwargs({ body: requestPayload, model });
|
|
124670
|
+
}
|
|
124405
124671
|
if (requestPayload.stream === true) {
|
|
124406
124672
|
const streamOptions = requestPayload.stream_options;
|
|
124407
124673
|
const normalizedStreamOptions = isPlainObject$3(streamOptions)
|
|
@@ -124600,7 +124866,7 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
|
|
|
124600
124866
|
const engineType = conduitConfiguration.engineConfig?.type ?? null;
|
|
124601
124867
|
const engineConfig = conduitConfiguration.engineConfig?.config ?? null;
|
|
124602
124868
|
const effectiveBody = modelManager.model.multimodalEnabled ? body : stripImagesFromBody(body);
|
|
124603
|
-
const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(effectiveBody);
|
|
124869
|
+
const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(effectiveBody, { model: modelManager.model, path });
|
|
124604
124870
|
const requestStartedAt = Date.now();
|
|
124605
124871
|
const requestBody = JSON.parse(serializedBody);
|
|
124606
124872
|
const streamRequested = requestBody.stream === true;
|
|
@@ -124986,6 +125252,17 @@ function translateAnthropicRequestToOpenAI(body) {
|
|
|
124986
125252
|
result.top_p = parsed.top_p;
|
|
124987
125253
|
if (Array.isArray(parsed.stop_sequences))
|
|
124988
125254
|
result.stop = parsed.stop_sequences;
|
|
125255
|
+
if (parsed.thinking && typeof parsed.thinking === "object") {
|
|
125256
|
+
const thinking = parsed.thinking;
|
|
125257
|
+
if (thinking.type === "enabled") {
|
|
125258
|
+
const budgetTokens = typeof thinking.budget_tokens === "number" ? thinking.budget_tokens : null;
|
|
125259
|
+
result.chat_template_kwargs = {
|
|
125260
|
+
reasoning_effort: budgetTokens
|
|
125261
|
+
? reasoningEffortForAnthropicBudget(budgetTokens)
|
|
125262
|
+
: "high"
|
|
125263
|
+
};
|
|
125264
|
+
}
|
|
125265
|
+
}
|
|
124989
125266
|
if (Array.isArray(parsed.tools)) {
|
|
124990
125267
|
result.tools = parsed.tools.map((tool) => {
|
|
124991
125268
|
const t = tool;
|
|
@@ -125257,11 +125534,13 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
|
|
|
125257
125534
|
const requestStartedAt = Date.now();
|
|
125258
125535
|
const requestBody = JSON.parse(serializedBody);
|
|
125259
125536
|
const streamRequested = requestBody.stream === true;
|
|
125260
|
-
const
|
|
125261
|
-
|
|
125262
|
-
|
|
125263
|
-
|
|
125264
|
-
|
|
125537
|
+
const translated = needsTranslation ? translateAnthropicRequestToOpenAI(serializedBody) : null;
|
|
125538
|
+
const targetPath = translated?.path ?? "/v1/messages";
|
|
125539
|
+
const targetBody = translated
|
|
125540
|
+
? JSON.stringify(applyChatTemplateKwargs({
|
|
125541
|
+
body: JSON.parse(translated.body),
|
|
125542
|
+
model: modelManager.model
|
|
125543
|
+
}))
|
|
125265
125544
|
: serializedBody;
|
|
125266
125545
|
const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
|
|
125267
125546
|
const promptTokens = normalizeTokenCount(usage?.inputTokens);
|
|
@@ -135944,13 +136223,14 @@ async function createApplication({ abortController, apiClient, configuration, lo
|
|
|
135944
136223
|
});
|
|
135945
136224
|
conduitStateReportManager.reportStateChange();
|
|
135946
136225
|
};
|
|
135947
|
-
const setOnlineState = () => {
|
|
135948
|
-
if (conduitStateManager.getState().state === "online") {
|
|
136226
|
+
const setOnlineState = ({ warnings } = {}) => {
|
|
136227
|
+
if (conduitStateManager.getState().state === "online" && warnings === undefined) {
|
|
135949
136228
|
return;
|
|
135950
136229
|
}
|
|
135951
136230
|
conduitStateManager.setState({
|
|
135952
136231
|
modelName,
|
|
135953
|
-
state: "online"
|
|
136232
|
+
state: "online",
|
|
136233
|
+
...(warnings !== undefined ? { warnings } : {})
|
|
135954
136234
|
});
|
|
135955
136235
|
conduitStateReportManager.reportStateChange();
|
|
135956
136236
|
};
|
|
@@ -135972,6 +136252,23 @@ async function createApplication({ abortController, apiClient, configuration, lo
|
|
|
135972
136252
|
});
|
|
135973
136253
|
modelManager.on("engineReady", () => {
|
|
135974
136254
|
setOnlineState();
|
|
136255
|
+
const readyModelManager = modelManager;
|
|
136256
|
+
readyModelManager
|
|
136257
|
+
.verifyChatTemplate()
|
|
136258
|
+
.then(warning => {
|
|
136259
|
+
if (modelManager !== readyModelManager) {
|
|
136260
|
+
return;
|
|
136261
|
+
}
|
|
136262
|
+
if (warning) {
|
|
136263
|
+
logger.warn("Chat template verification", { warning });
|
|
136264
|
+
setOnlineState({ warnings: [warning] });
|
|
136265
|
+
}
|
|
136266
|
+
})
|
|
136267
|
+
.catch(error => {
|
|
136268
|
+
logger.warn("Chat template verification failed", {
|
|
136269
|
+
error: asError(error)
|
|
136270
|
+
});
|
|
136271
|
+
});
|
|
135975
136272
|
});
|
|
135976
136273
|
modelManager.on("engineTerminated", () => {
|
|
135977
136274
|
if (stopRequestedByControl) {
|