@infersec/conduit 1.99.2 → 1.101.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2450 -2153
- package/dist/cli.sea.cjs +2448 -2151
- package/dist/modelManagement/ModelManager.d.ts +1 -0
- package/dist/modelManagement/chatTemplate.d.ts +41 -0
- package/dist/requestHandlers/createConduitOpenAIAPIReferenceHandlers.d.ts +3 -1
- package/dist/utils/openai.d.ts +18 -1
- package/package.json +1 -1
package/dist/cli.sea.cjs
CHANGED
|
@@ -42,10 +42,10 @@ var require$$1$8 = require('node:async_hooks');
|
|
|
42
42
|
var require$$1$9 = require('node:console');
|
|
43
43
|
var require$$0$m = require('node:fs/promises');
|
|
44
44
|
var require$$2$5 = require('node:timers');
|
|
45
|
-
var node_string_decoder = require('node:string_decoder');
|
|
46
|
-
var promises = require('node:stream/promises');
|
|
47
45
|
require('fs/promises');
|
|
48
46
|
require('stream/promises');
|
|
47
|
+
var node_string_decoder = require('node:string_decoder');
|
|
48
|
+
var promises = require('node:stream/promises');
|
|
49
49
|
var os = require('node:os');
|
|
50
50
|
var tty = require('node:tty');
|
|
51
51
|
var require$$3$9 = require('child_process');
|
|
@@ -20001,7 +20001,39 @@ object$1({
|
|
|
20001
20001
|
supportsVision: boolean$1()
|
|
20002
20002
|
});
|
|
20003
20003
|
const LLMModelTaskTypeSchema = _enum$1(["text-generation", "embeddings"]);
|
|
20004
|
+
const ReasoningEffortSchema = _enum$1(["none", "minimal", "low", "medium", "high", "xhigh"]);
|
|
20005
|
+
const CHAT_TEMPLATE_DEFAULT_FILE_PATH = "chat_template.jinja";
|
|
20006
|
+
const CHAT_TEMPLATE_LOCAL_FILE_NAME = "infersec-chat-template.jinja";
|
|
20007
|
+
const ChatTemplateOverrideSchema = discriminatedUnion("type", [
|
|
20008
|
+
object$1({
|
|
20009
|
+
filePath: string$2().min(1).default(CHAT_TEMPLATE_DEFAULT_FILE_PATH),
|
|
20010
|
+
repo: string$2().min(1),
|
|
20011
|
+
type: literal("huggingface")
|
|
20012
|
+
}),
|
|
20013
|
+
object$1({
|
|
20014
|
+
content: string$2().min(1),
|
|
20015
|
+
type: literal("inline")
|
|
20016
|
+
})
|
|
20017
|
+
]);
|
|
20018
|
+
const ThinkingConfigSchema = object$1({
|
|
20019
|
+
enabled: boolean$1().optional(),
|
|
20020
|
+
effort: ReasoningEffortSchema.optional()
|
|
20021
|
+
});
|
|
20022
|
+
const ANTHROPIC_THINKING_BUDGET_EFFORT_TIERS = [
|
|
20023
|
+
{ effort: "low", maxBudgetTokens: 2048 },
|
|
20024
|
+
{ effort: "medium", maxBudgetTokens: 8192 },
|
|
20025
|
+
{ effort: "high", maxBudgetTokens: Number.POSITIVE_INFINITY }
|
|
20026
|
+
];
|
|
20027
|
+
function reasoningEffortForAnthropicBudget(budgetTokens) {
|
|
20028
|
+
for (const tier of ANTHROPIC_THINKING_BUDGET_EFFORT_TIERS) {
|
|
20029
|
+
if (budgetTokens <= tier.maxBudgetTokens) {
|
|
20030
|
+
return tier.effort;
|
|
20031
|
+
}
|
|
20032
|
+
}
|
|
20033
|
+
return "high";
|
|
20034
|
+
}
|
|
20004
20035
|
const LLMModelSchema = object$1({
|
|
20036
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable().optional(),
|
|
20005
20037
|
format: LLMModelFormatSchema,
|
|
20006
20038
|
id: string$2().min(1),
|
|
20007
20039
|
multimodalEnabled: boolean$1(),
|
|
@@ -20016,7 +20048,8 @@ const LLMModelSchema = object$1({
|
|
|
20016
20048
|
type: literal("huggingface")
|
|
20017
20049
|
})
|
|
20018
20050
|
]),
|
|
20019
|
-
taskType: LLMModelTaskTypeSchema
|
|
20051
|
+
taskType: LLMModelTaskTypeSchema,
|
|
20052
|
+
thinkingConfig: ThinkingConfigSchema.nullable().optional()
|
|
20020
20053
|
});
|
|
20021
20054
|
object$1({
|
|
20022
20055
|
filePath: string$2().min(1),
|
|
@@ -20135,7 +20168,8 @@ const ConduitStateSchema = z
|
|
|
20135
20168
|
])
|
|
20136
20169
|
.and(z.object({
|
|
20137
20170
|
activeRequestCount: z.number().int().nonnegative().optional(),
|
|
20138
|
-
timestamp: z.string().datetime()
|
|
20171
|
+
timestamp: z.string().datetime(),
|
|
20172
|
+
warnings: z.array(z.string()).optional()
|
|
20139
20173
|
}));
|
|
20140
20174
|
const ConduitState = z.preprocess(value => {
|
|
20141
20175
|
if (value === null) {
|
|
@@ -20674,6 +20708,10 @@ const ChatCompletionMessageSchema = object$1({
|
|
|
20674
20708
|
const ChatCompletionCreateParamsSchema = object$1({
|
|
20675
20709
|
messages: array(ChatCompletionMessageParamSchema),
|
|
20676
20710
|
model: string$2(),
|
|
20711
|
+
chat_template_kwargs: record(string$2(), unknown())
|
|
20712
|
+
.nullable()
|
|
20713
|
+
.optional()
|
|
20714
|
+
.describe("Additional chat template variables passed to the model's chat template at render time (e.g. reasoning_effort)"),
|
|
20677
20715
|
frequency_penalty: number$1().min(-2).max(2).nullable().optional(),
|
|
20678
20716
|
function_call: union([literal("none"), literal("auto"), object$1({ name: string$2() })])
|
|
20679
20717
|
.optional(),
|
|
@@ -20689,6 +20727,9 @@ const ChatCompletionCreateParamsSchema = object$1({
|
|
|
20689
20727
|
max_tokens: number$1().positive().nullable().optional(),
|
|
20690
20728
|
n: number$1().positive().nullable().optional(),
|
|
20691
20729
|
presence_penalty: number$1().min(-2).max(2).nullable().optional(),
|
|
20730
|
+
reasoning_effort: ReasoningEffortSchema.nullable()
|
|
20731
|
+
.optional()
|
|
20732
|
+
.describe("Reasoning effort hint for thinking models. Forwarded to the chat template when one is configured"),
|
|
20692
20733
|
response_format: object$1({
|
|
20693
20734
|
type: _enum$1(["text", "json_object"])
|
|
20694
20735
|
})
|
|
@@ -20970,6 +21011,7 @@ RoutingMethod.FirstAvailable;
|
|
|
20970
21011
|
});
|
|
20971
21012
|
|
|
20972
21013
|
object$1({
|
|
21014
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable().optional(),
|
|
20973
21015
|
format: LLMModelFormatSchema,
|
|
20974
21016
|
multimodalEnabled: boolean$1().optional(),
|
|
20975
21017
|
name: ResourceNameSchema,
|
|
@@ -20979,7 +21021,8 @@ object$1({
|
|
|
20979
21021
|
.refine(value => value.includes("/"), {
|
|
20980
21022
|
message: "Slug must be fully qualified (owner/repo)"
|
|
20981
21023
|
}),
|
|
20982
|
-
taskType: LLMModelTaskTypeSchema.optional()
|
|
21024
|
+
taskType: LLMModelTaskTypeSchema.optional(),
|
|
21025
|
+
thinkingConfig: ThinkingConfigSchema.nullable().optional()
|
|
20983
21026
|
});
|
|
20984
21027
|
object$1({
|
|
20985
21028
|
results: array(object$1({
|
|
@@ -20995,6 +21038,7 @@ object$1({
|
|
|
20995
21038
|
}))
|
|
20996
21039
|
});
|
|
20997
21040
|
object$1({
|
|
21041
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable(),
|
|
20998
21042
|
created: string$2(),
|
|
20999
21043
|
id: ULIDSchema,
|
|
21000
21044
|
modelFormat: LLMModelFormatSchema,
|
|
@@ -21012,12 +21056,15 @@ object$1({
|
|
|
21012
21056
|
updated: string$2()
|
|
21013
21057
|
})),
|
|
21014
21058
|
taskType: LLMModelTaskTypeSchema,
|
|
21059
|
+
thinkingConfig: ThinkingConfigSchema.nullable(),
|
|
21015
21060
|
updated: string$2()
|
|
21016
21061
|
});
|
|
21017
21062
|
object$1({
|
|
21063
|
+
chatTemplate: ChatTemplateOverrideSchema.nullable().optional(),
|
|
21018
21064
|
multimodalEnabled: boolean$1().optional(),
|
|
21019
21065
|
name: ResourceNameSchema.optional(),
|
|
21020
|
-
taskType: LLMModelTaskTypeSchema.optional()
|
|
21066
|
+
taskType: LLMModelTaskTypeSchema.optional(),
|
|
21067
|
+
thinkingConfig: ThinkingConfigSchema.nullable().optional()
|
|
21021
21068
|
});
|
|
21022
21069
|
object$1({
|
|
21023
21070
|
success: literal(true)
|
|
@@ -112947,653 +112994,1858 @@ var createRouter = /*@__PURE__*/getDefaultExportFromCjs(expressPromiseRouterExpo
|
|
|
112947
112994
|
|
|
112948
112995
|
const SERVED_MODEL_NAME = "default";
|
|
112949
112996
|
|
|
112950
|
-
|
|
112951
|
-
"api-key",
|
|
112952
|
-
"auth-token",
|
|
112953
|
-
"hf-token",
|
|
112954
|
-
"key",
|
|
112955
|
-
"password",
|
|
112956
|
-
"secret",
|
|
112957
|
-
"token"
|
|
112958
|
-
]);
|
|
112959
|
-
function redactSecretArgs(args) {
|
|
112960
|
-
return args.map((arg, index) => {
|
|
112961
|
-
const equalsIndex = arg.indexOf("=");
|
|
112962
|
-
if (arg.startsWith("--") && equalsIndex > 0) {
|
|
112963
|
-
const flag = arg.slice(2, equalsIndex);
|
|
112964
|
-
if (SECRET_FLAGS.has(flag)) {
|
|
112965
|
-
return `${arg.slice(0, equalsIndex + 1)}***`;
|
|
112966
|
-
}
|
|
112967
|
-
}
|
|
112968
|
-
const previous = args[index - 1];
|
|
112969
|
-
if (previous &&
|
|
112970
|
-
previous.startsWith("--") &&
|
|
112971
|
-
!previous.includes("=") &&
|
|
112972
|
-
SECRET_FLAGS.has(previous.slice(2))) {
|
|
112973
|
-
return "***";
|
|
112974
|
-
}
|
|
112975
|
-
return arg;
|
|
112976
|
-
});
|
|
112977
|
-
}
|
|
112978
|
-
async function createEngineProcess({ args, bin, logger }) {
|
|
112979
|
-
logger.info("Starting engine process", {
|
|
112980
|
-
command: { args: redactSecretArgs(args), bin }
|
|
112981
|
-
});
|
|
112982
|
-
const processManager = new ProcessManager({ args, command: bin });
|
|
112983
|
-
await processManager.start();
|
|
112984
|
-
return processManager;
|
|
112985
|
-
}
|
|
112997
|
+
// src/lib/cache-management.ts
|
|
112986
112998
|
|
|
112987
|
-
|
|
112988
|
-
|
|
112989
|
-
|
|
112990
|
-
|
|
112999
|
+
// src/consts.ts
|
|
113000
|
+
var HUB_URL = "https://huggingface.co";
|
|
113001
|
+
|
|
113002
|
+
// src/error.ts
|
|
113003
|
+
async function createApiError(response, opts) {
|
|
113004
|
+
const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
|
|
113005
|
+
error.message = `Api error with status ${error.statusCode}${""}`;
|
|
113006
|
+
const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". ");
|
|
113007
|
+
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
|
|
113008
|
+
const json = await response.json();
|
|
113009
|
+
error.message = json.error || json.message || error.message;
|
|
113010
|
+
if (json.error_description) {
|
|
113011
|
+
error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description;
|
|
112991
113012
|
}
|
|
112992
|
-
|
|
113013
|
+
error.data = json;
|
|
113014
|
+
} else {
|
|
113015
|
+
error.data = { message: await response.text() };
|
|
113016
|
+
}
|
|
113017
|
+
error.message += `. ${trailer}`;
|
|
113018
|
+
throw error;
|
|
112993
113019
|
}
|
|
112994
|
-
|
|
112995
|
-
|
|
112996
|
-
|
|
112997
|
-
|
|
112998
|
-
|
|
112999
|
-
|
|
113000
|
-
|
|
113001
|
-
|
|
113002
|
-
|
|
113003
|
-
|
|
113004
|
-
|
|
113005
|
-
|
|
113006
|
-
|
|
113007
|
-
|
|
113008
|
-
|
|
113009
|
-
|
|
113010
|
-
|
|
113011
|
-
|
|
113012
|
-
|
|
113013
|
-
|
|
113014
|
-
|
|
113015
|
-
|
|
113016
|
-
|
|
113017
|
-
|
|
113018
|
-
|
|
113019
|
-
|
|
113020
|
-
|
|
113021
|
-
|
|
113022
|
-
|
|
113023
|
-
|
|
113024
|
-
|
|
113020
|
+
var HubApiError = class extends Error {
|
|
113021
|
+
statusCode;
|
|
113022
|
+
url;
|
|
113023
|
+
requestId;
|
|
113024
|
+
data;
|
|
113025
|
+
constructor(url, statusCode, requestId, message) {
|
|
113026
|
+
super(message);
|
|
113027
|
+
this.statusCode = statusCode;
|
|
113028
|
+
this.requestId = requestId;
|
|
113029
|
+
this.url = url;
|
|
113030
|
+
}
|
|
113031
|
+
};
|
|
113032
|
+
var InvalidApiResponseFormatError = class extends Error {
|
|
113033
|
+
};
|
|
113034
|
+
|
|
113035
|
+
// src/utils/checkCredentials.ts
|
|
113036
|
+
function checkAccessToken(accessToken) {
|
|
113037
|
+
if (!accessToken.startsWith("hf_")) {
|
|
113038
|
+
throw new TypeError("Your access token must start with 'hf_'");
|
|
113039
|
+
}
|
|
113040
|
+
}
|
|
113041
|
+
function checkCredentials(params) {
|
|
113042
|
+
if (params.accessToken) {
|
|
113043
|
+
checkAccessToken(params.accessToken);
|
|
113044
|
+
return params.accessToken;
|
|
113045
|
+
}
|
|
113046
|
+
if (params.credentials?.accessToken) {
|
|
113047
|
+
checkAccessToken(params.credentials.accessToken);
|
|
113048
|
+
return params.credentials.accessToken;
|
|
113049
|
+
}
|
|
113050
|
+
}
|
|
113051
|
+
|
|
113052
|
+
// src/utils/toRepoId.ts
|
|
113053
|
+
function toRepoId(repo) {
|
|
113054
|
+
if (typeof repo !== "string") {
|
|
113055
|
+
return repo;
|
|
113056
|
+
}
|
|
113057
|
+
if (repo.startsWith("model/") || repo.startsWith("models/")) {
|
|
113058
|
+
throw new TypeError(
|
|
113059
|
+
"A repo designation for a model should not start with 'models/', directly specify the model namespace / name"
|
|
113060
|
+
);
|
|
113061
|
+
}
|
|
113062
|
+
if (repo.startsWith("space/")) {
|
|
113063
|
+
throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'");
|
|
113064
|
+
}
|
|
113065
|
+
if (repo.startsWith("dataset/")) {
|
|
113066
|
+
throw new TypeError("Datasets should start with 'dataset/', plural, not 'dataset/'");
|
|
113067
|
+
}
|
|
113068
|
+
const slashes = repo.split("/").length - 1;
|
|
113069
|
+
if (repo.startsWith("spaces/")) {
|
|
113070
|
+
if (slashes !== 2) {
|
|
113071
|
+
throw new TypeError("Space Id must include namespace and name of the space");
|
|
113025
113072
|
}
|
|
113026
|
-
|
|
113027
|
-
|
|
113073
|
+
return {
|
|
113074
|
+
type: "space",
|
|
113075
|
+
name: repo.slice("spaces/".length)
|
|
113076
|
+
};
|
|
113077
|
+
}
|
|
113078
|
+
if (repo.startsWith("datasets/")) {
|
|
113079
|
+
if (slashes > 2) {
|
|
113080
|
+
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
113028
113081
|
}
|
|
113029
|
-
return
|
|
113082
|
+
return {
|
|
113083
|
+
type: "dataset",
|
|
113084
|
+
name: repo.slice("datasets/".length)
|
|
113085
|
+
};
|
|
113086
|
+
}
|
|
113087
|
+
if (slashes > 1) {
|
|
113088
|
+
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
113089
|
+
}
|
|
113090
|
+
return {
|
|
113091
|
+
type: "model",
|
|
113092
|
+
name: repo
|
|
113093
|
+
};
|
|
113030
113094
|
}
|
|
113095
|
+
new Promise((r) => {
|
|
113096
|
+
});
|
|
113031
113097
|
|
|
113032
|
-
|
|
113033
|
-
|
|
113034
|
-
|
|
113035
|
-
|
|
113036
|
-
|
|
113037
|
-
|
|
113038
|
-
|
|
113039
|
-
|
|
113040
|
-
|
|
113041
|
-
|
|
113042
|
-
|
|
113043
|
-
|
|
113044
|
-
|
|
113045
|
-
|
|
113046
|
-
|
|
113098
|
+
// src/utils/combineUint8Arrays.ts
|
|
113099
|
+
function combineUint8Arrays(a, b) {
|
|
113100
|
+
const aLength = a.length;
|
|
113101
|
+
const combinedBytes = new Uint8Array(aLength + b.length);
|
|
113102
|
+
combinedBytes.set(a);
|
|
113103
|
+
combinedBytes.set(b, aLength);
|
|
113104
|
+
return combinedBytes;
|
|
113105
|
+
}
|
|
113106
|
+
function readU64(b, n) {
|
|
113107
|
+
let x = 0;
|
|
113108
|
+
x |= b[n++] << 0;
|
|
113109
|
+
x |= b[n++] << 8;
|
|
113110
|
+
x |= b[n++] << 16;
|
|
113111
|
+
x |= b[n++] << 24;
|
|
113112
|
+
x |= b[n++] << 32;
|
|
113113
|
+
x |= b[n++] << 40;
|
|
113114
|
+
x |= b[n++] << 48;
|
|
113115
|
+
x |= b[n++] << 56;
|
|
113116
|
+
return x;
|
|
113117
|
+
}
|
|
113118
|
+
function readU32(b, n) {
|
|
113119
|
+
let x = 0;
|
|
113120
|
+
x |= b[n++] << 0;
|
|
113121
|
+
x |= b[n++] << 8;
|
|
113122
|
+
x |= b[n++] << 16;
|
|
113123
|
+
x |= b[n++] << 24;
|
|
113124
|
+
return x;
|
|
113125
|
+
}
|
|
113126
|
+
|
|
113127
|
+
// src/vendor/lz4js/index.ts
|
|
113128
|
+
var minMatch = 4;
|
|
113129
|
+
var hashSize = 1 << 16;
|
|
113130
|
+
makeHashTable();
|
|
113131
|
+
var magicNum = 407708164;
|
|
113132
|
+
var fdContentChksum = 4;
|
|
113133
|
+
var fdContentSize = 8;
|
|
113134
|
+
var fdBlockChksum = 16;
|
|
113135
|
+
var fdVersion = 64;
|
|
113136
|
+
var fdVersionMask = 192;
|
|
113137
|
+
var bsUncompressed = 2147483648;
|
|
113138
|
+
var bsShift = 4;
|
|
113139
|
+
var bsMask = 7;
|
|
113140
|
+
var bsMap = {
|
|
113141
|
+
4: 65536,
|
|
113142
|
+
5: 262144,
|
|
113143
|
+
6: 1048576,
|
|
113144
|
+
7: 4194304
|
|
113047
113145
|
};
|
|
113048
|
-
|
|
113049
|
-
|
|
113050
|
-
|
|
113051
|
-
|
|
113052
|
-
|
|
113053
|
-
|
|
113054
|
-
|
|
113055
|
-
return [ai, bi];
|
|
113056
|
-
}
|
|
113057
|
-
begs = [];
|
|
113058
|
-
left = str.length;
|
|
113059
|
-
while (i >= 0 && !result) {
|
|
113060
|
-
if (i === ai) {
|
|
113061
|
-
begs.push(i);
|
|
113062
|
-
ai = str.indexOf(a, i + 1);
|
|
113063
|
-
}
|
|
113064
|
-
else if (begs.length === 1) {
|
|
113065
|
-
const r = begs.pop();
|
|
113066
|
-
if (r !== undefined)
|
|
113067
|
-
result = [r, bi];
|
|
113068
|
-
}
|
|
113069
|
-
else {
|
|
113070
|
-
beg = begs.pop();
|
|
113071
|
-
if (beg !== undefined && beg < left) {
|
|
113072
|
-
left = beg;
|
|
113073
|
-
right = bi;
|
|
113074
|
-
}
|
|
113075
|
-
bi = str.indexOf(b, i + 1);
|
|
113076
|
-
}
|
|
113077
|
-
i = ai < bi && ai >= 0 ? ai : bi;
|
|
113078
|
-
}
|
|
113079
|
-
if (begs.length && right !== undefined) {
|
|
113080
|
-
result = [left, right];
|
|
113081
|
-
}
|
|
113146
|
+
function makeHashTable() {
|
|
113147
|
+
try {
|
|
113148
|
+
return new Uint32Array(hashSize);
|
|
113149
|
+
} catch (error) {
|
|
113150
|
+
const hashTable2 = new Array(hashSize);
|
|
113151
|
+
for (let i = 0; i < hashSize; i++) {
|
|
113152
|
+
hashTable2[i] = 0;
|
|
113082
113153
|
}
|
|
113083
|
-
return
|
|
113084
|
-
}
|
|
113085
|
-
|
|
113086
|
-
const escSlash = '\0SLASH' + Math.random() + '\0';
|
|
113087
|
-
const escOpen = '\0OPEN' + Math.random() + '\0';
|
|
113088
|
-
const escClose = '\0CLOSE' + Math.random() + '\0';
|
|
113089
|
-
const escComma = '\0COMMA' + Math.random() + '\0';
|
|
113090
|
-
const escPeriod = '\0PERIOD' + Math.random() + '\0';
|
|
113091
|
-
const escSlashPattern = new RegExp(escSlash, 'g');
|
|
113092
|
-
const escOpenPattern = new RegExp(escOpen, 'g');
|
|
113093
|
-
const escClosePattern = new RegExp(escClose, 'g');
|
|
113094
|
-
const escCommaPattern = new RegExp(escComma, 'g');
|
|
113095
|
-
const escPeriodPattern = new RegExp(escPeriod, 'g');
|
|
113096
|
-
const slashPattern = /\\\\/g;
|
|
113097
|
-
const openPattern = /\\{/g;
|
|
113098
|
-
const closePattern = /\\}/g;
|
|
113099
|
-
const commaPattern = /\\,/g;
|
|
113100
|
-
const periodPattern = /\\\./g;
|
|
113101
|
-
const EXPANSION_MAX = 100_000;
|
|
113102
|
-
// `EXPANSION_MAX` caps the *number* of expansions, but not their length. An
|
|
113103
|
-
// input like `'{a,b}'.repeat(1500)` stays under that count - its output is
|
|
113104
|
-
// truncated to 100k results - while making every result ~1500 characters
|
|
113105
|
-
// long. The result set, and the intermediate arrays built while combining
|
|
113106
|
-
// brace sets, then grow large enough to exhaust memory and crash the process
|
|
113107
|
-
// (CVE-2026-14257). `EXPANSION_MAX_LENGTH` bounds the total number of
|
|
113108
|
-
// characters the accumulator may hold at any point, so memory stays flat no
|
|
113109
|
-
// matter how many brace groups are chained. The limit sits well above any
|
|
113110
|
-
// realistic expansion (100k results hitting `EXPANSION_MAX` measure ~1M
|
|
113111
|
-
// characters) so legitimate input is unaffected.
|
|
113112
|
-
const EXPANSION_MAX_LENGTH = 4_000_000;
|
|
113113
|
-
function numeric(str) {
|
|
113114
|
-
return !isNaN(str) ? parseInt(str, 10) : str.charCodeAt(0);
|
|
113154
|
+
return hashTable2;
|
|
113155
|
+
}
|
|
113115
113156
|
}
|
|
113116
|
-
function
|
|
113117
|
-
|
|
113118
|
-
.replace(slashPattern, escSlash)
|
|
113119
|
-
.replace(openPattern, escOpen)
|
|
113120
|
-
.replace(closePattern, escClose)
|
|
113121
|
-
.replace(commaPattern, escComma)
|
|
113122
|
-
.replace(periodPattern, escPeriod);
|
|
113157
|
+
function makeBuffer(size) {
|
|
113158
|
+
return new Uint8Array(size);
|
|
113123
113159
|
}
|
|
113124
|
-
function
|
|
113125
|
-
|
|
113126
|
-
.replace(escSlashPattern, '\\')
|
|
113127
|
-
.replace(escOpenPattern, '{')
|
|
113128
|
-
.replace(escClosePattern, '}')
|
|
113129
|
-
.replace(escCommaPattern, ',')
|
|
113130
|
-
.replace(escPeriodPattern, '.');
|
|
113160
|
+
function sliceArray(array, start, end) {
|
|
113161
|
+
return array.slice(start, end);
|
|
113131
113162
|
}
|
|
113132
|
-
|
|
113133
|
-
|
|
113134
|
-
|
|
113135
|
-
|
|
113136
|
-
|
|
113137
|
-
|
|
113138
|
-
|
|
113139
|
-
|
|
113163
|
+
function decompressBound(src) {
|
|
113164
|
+
let sIndex = 0;
|
|
113165
|
+
if (readU32(src, sIndex) !== magicNum) {
|
|
113166
|
+
throw new Error("invalid magic number");
|
|
113167
|
+
}
|
|
113168
|
+
sIndex += 4;
|
|
113169
|
+
const descriptor = src[sIndex++];
|
|
113170
|
+
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
113171
|
+
throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask));
|
|
113172
|
+
}
|
|
113173
|
+
const useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
113174
|
+
const useContentSize = (descriptor & fdContentSize) !== 0;
|
|
113175
|
+
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
113176
|
+
if (bsMap[bsIdx] === void 0) {
|
|
113177
|
+
throw new Error("invalid block size " + bsIdx);
|
|
113178
|
+
}
|
|
113179
|
+
const maxBlockSize = bsMap[bsIdx];
|
|
113180
|
+
if (useContentSize) {
|
|
113181
|
+
return readU64(src, sIndex);
|
|
113182
|
+
}
|
|
113183
|
+
sIndex++;
|
|
113184
|
+
let maxSize = 0;
|
|
113185
|
+
while (true) {
|
|
113186
|
+
let blockSize = readU32(src, sIndex);
|
|
113187
|
+
sIndex += 4;
|
|
113188
|
+
if (blockSize & bsUncompressed) {
|
|
113189
|
+
blockSize &= ~bsUncompressed;
|
|
113190
|
+
maxSize += blockSize;
|
|
113191
|
+
} else if (blockSize > 0) {
|
|
113192
|
+
maxSize += maxBlockSize;
|
|
113140
113193
|
}
|
|
113141
|
-
|
|
113142
|
-
|
|
113143
|
-
if (!m) {
|
|
113144
|
-
return str.split(',');
|
|
113194
|
+
if (blockSize === 0) {
|
|
113195
|
+
return maxSize;
|
|
113145
113196
|
}
|
|
113146
|
-
|
|
113147
|
-
|
|
113148
|
-
p[p.length - 1] += '{' + body + '}';
|
|
113149
|
-
const postParts = parseCommaParts(post);
|
|
113150
|
-
if (post.length) {
|
|
113151
|
-
p[p.length - 1] += postParts.shift();
|
|
113152
|
-
p.push.apply(p, postParts);
|
|
113197
|
+
if (useBlockSum) {
|
|
113198
|
+
sIndex += 4;
|
|
113153
113199
|
}
|
|
113154
|
-
|
|
113155
|
-
|
|
113200
|
+
sIndex += blockSize;
|
|
113201
|
+
}
|
|
113156
113202
|
}
|
|
113157
|
-
function
|
|
113158
|
-
|
|
113159
|
-
|
|
113203
|
+
function decompressBlock(src, dst, sIndex, sLength, dIndex) {
|
|
113204
|
+
let mLength, mOffset, sEnd, n, i;
|
|
113205
|
+
const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0;
|
|
113206
|
+
sEnd = sIndex + sLength;
|
|
113207
|
+
while (sIndex < sEnd) {
|
|
113208
|
+
const token = src[sIndex++];
|
|
113209
|
+
let literalCount = token >> 4;
|
|
113210
|
+
if (literalCount > 0) {
|
|
113211
|
+
if (literalCount === 15) {
|
|
113212
|
+
while (true) {
|
|
113213
|
+
literalCount += src[sIndex];
|
|
113214
|
+
if (src[sIndex++] !== 255) {
|
|
113215
|
+
break;
|
|
113216
|
+
}
|
|
113217
|
+
}
|
|
113218
|
+
}
|
|
113219
|
+
for (n = sIndex + literalCount; sIndex < n; ) {
|
|
113220
|
+
dst[dIndex++] = src[sIndex++];
|
|
113221
|
+
}
|
|
113160
113222
|
}
|
|
113161
|
-
|
|
113162
|
-
|
|
113163
|
-
// Anything starting with {} will have the first two bytes preserved
|
|
113164
|
-
// but *only* at the top level, so {},a}b will not expand to anything,
|
|
113165
|
-
// but a{},b}c will be expanded to [a}c,abc].
|
|
113166
|
-
// One could argue that this is a bug in Bash, but since the goal of
|
|
113167
|
-
// this module is to match Bash's rules, we escape a leading {}
|
|
113168
|
-
if (str.slice(0, 2) === '{}') {
|
|
113169
|
-
str = '\\{\\}' + str.slice(2);
|
|
113223
|
+
if (sIndex >= sEnd) {
|
|
113224
|
+
break;
|
|
113170
113225
|
}
|
|
113171
|
-
|
|
113172
|
-
|
|
113173
|
-
|
|
113174
|
-
|
|
113175
|
-
|
|
113176
|
-
|
|
113177
|
-
|
|
113178
|
-
}
|
|
113179
|
-
function lte(i, y) {
|
|
113180
|
-
return i <= y;
|
|
113181
|
-
}
|
|
113182
|
-
function gte(i, y) {
|
|
113183
|
-
return i >= y;
|
|
113184
|
-
}
|
|
113185
|
-
// Build `{ acc[a] + pre + values[v] }` for every combination, capping the
|
|
113186
|
-
// number of results at `max` and the total number of characters at `maxLength`.
|
|
113187
|
-
// This is the one place output grows, so bounding it here keeps the single
|
|
113188
|
-
// accumulator - and therefore memory - flat regardless of how many brace groups
|
|
113189
|
-
// are combined (CVE-2026-14257).
|
|
113190
|
-
function combine(acc, pre, values, max, maxLength, dropEmpties) {
|
|
113191
|
-
const out = [];
|
|
113192
|
-
let length = 0;
|
|
113193
|
-
for (let a = 0; a < acc.length; a++) {
|
|
113194
|
-
for (let v = 0; v < values.length; v++) {
|
|
113195
|
-
if (out.length >= max)
|
|
113196
|
-
return out;
|
|
113197
|
-
const expansion = acc[a] + pre + values[v];
|
|
113198
|
-
// Bash drops empty results at the top level. Skip them before they count
|
|
113199
|
-
// against `max`, so `max` bounds the number of *kept* results.
|
|
113200
|
-
if (dropEmpties && !expansion)
|
|
113201
|
-
continue;
|
|
113202
|
-
if (length + expansion.length > maxLength)
|
|
113203
|
-
return out;
|
|
113204
|
-
out.push(expansion);
|
|
113205
|
-
length += expansion.length;
|
|
113226
|
+
mLength = token & 15;
|
|
113227
|
+
mOffset = src[sIndex++] | src[sIndex++] << 8;
|
|
113228
|
+
if (mLength === 15) {
|
|
113229
|
+
while (true) {
|
|
113230
|
+
mLength += src[sIndex];
|
|
113231
|
+
if (src[sIndex++] !== 255) {
|
|
113232
|
+
break;
|
|
113206
113233
|
}
|
|
113234
|
+
}
|
|
113207
113235
|
}
|
|
113208
|
-
|
|
113236
|
+
mLength += minMatch;
|
|
113237
|
+
if (hasCopyWithin && mOffset === 1) {
|
|
113238
|
+
dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength);
|
|
113239
|
+
dIndex += mLength;
|
|
113240
|
+
} else if (hasCopyWithin && mOffset > mLength && mLength > 31) {
|
|
113241
|
+
dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength);
|
|
113242
|
+
dIndex += mLength;
|
|
113243
|
+
} else {
|
|
113244
|
+
for (i = dIndex - mOffset, n = i + mLength; i < n; ) {
|
|
113245
|
+
dst[dIndex++] = dst[i++] | 0;
|
|
113246
|
+
}
|
|
113247
|
+
}
|
|
113248
|
+
}
|
|
113249
|
+
return dIndex;
|
|
113209
113250
|
}
|
|
113210
|
-
|
|
113211
|
-
|
|
113212
|
-
|
|
113213
|
-
|
|
113214
|
-
|
|
113215
|
-
|
|
113216
|
-
|
|
113217
|
-
|
|
113218
|
-
|
|
113219
|
-
|
|
113251
|
+
function decompressFrame(src, dst) {
|
|
113252
|
+
let useBlockSum, useContentSum, useContentSize, descriptor;
|
|
113253
|
+
let sIndex = 0;
|
|
113254
|
+
let dIndex = 0;
|
|
113255
|
+
if (readU32(src, sIndex) !== magicNum) {
|
|
113256
|
+
throw new Error("invalid magic number");
|
|
113257
|
+
}
|
|
113258
|
+
sIndex += 4;
|
|
113259
|
+
descriptor = src[sIndex++];
|
|
113260
|
+
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
113261
|
+
throw new Error("incompatible descriptor version");
|
|
113262
|
+
}
|
|
113263
|
+
useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
113264
|
+
useContentSum = (descriptor & fdContentChksum) !== 0;
|
|
113265
|
+
useContentSize = (descriptor & fdContentSize) !== 0;
|
|
113266
|
+
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
113267
|
+
if (bsMap[bsIdx] === void 0) {
|
|
113268
|
+
throw new Error("invalid block size");
|
|
113269
|
+
}
|
|
113270
|
+
if (useContentSize) {
|
|
113271
|
+
sIndex += 8;
|
|
113272
|
+
}
|
|
113273
|
+
sIndex++;
|
|
113274
|
+
while (true) {
|
|
113275
|
+
var compSize;
|
|
113276
|
+
compSize = readU32(src, sIndex);
|
|
113277
|
+
sIndex += 4;
|
|
113278
|
+
if (compSize === 0) {
|
|
113279
|
+
break;
|
|
113220
113280
|
}
|
|
113221
|
-
|
|
113222
|
-
|
|
113223
|
-
const y = numeric(n[1]);
|
|
113224
|
-
const width = Math.max(n[0].length, n[1].length);
|
|
113225
|
-
let incr = n.length === 3 && n[2] !== undefined ?
|
|
113226
|
-
Math.max(Math.abs(numeric(n[2])), 1)
|
|
113227
|
-
: 1;
|
|
113228
|
-
let test = lte;
|
|
113229
|
-
const reverse = y < x;
|
|
113230
|
-
if (reverse) {
|
|
113231
|
-
incr *= -1;
|
|
113232
|
-
test = gte;
|
|
113281
|
+
if (useBlockSum) {
|
|
113282
|
+
sIndex += 4;
|
|
113233
113283
|
}
|
|
113234
|
-
|
|
113235
|
-
|
|
113236
|
-
|
|
113237
|
-
|
|
113238
|
-
|
|
113239
|
-
|
|
113240
|
-
|
|
113241
|
-
|
|
113242
|
-
}
|
|
113243
|
-
}
|
|
113244
|
-
else {
|
|
113245
|
-
c = String(i);
|
|
113246
|
-
if (pad) {
|
|
113247
|
-
const need = width - c.length;
|
|
113248
|
-
if (need > 0) {
|
|
113249
|
-
const z = new Array(need + 1).join('0');
|
|
113250
|
-
if (i < 0) {
|
|
113251
|
-
c = '-' + z + c.slice(1);
|
|
113252
|
-
}
|
|
113253
|
-
else {
|
|
113254
|
-
c = z + c;
|
|
113255
|
-
}
|
|
113256
|
-
}
|
|
113257
|
-
}
|
|
113258
|
-
}
|
|
113259
|
-
if (length + c.length > maxLength)
|
|
113260
|
-
break;
|
|
113261
|
-
N.push(c);
|
|
113262
|
-
length += c.length;
|
|
113284
|
+
if ((compSize & bsUncompressed) !== 0) {
|
|
113285
|
+
compSize &= ~bsUncompressed;
|
|
113286
|
+
for (let j = 0; j < compSize; j++) {
|
|
113287
|
+
dst[dIndex++] = src[sIndex++];
|
|
113288
|
+
}
|
|
113289
|
+
} else {
|
|
113290
|
+
dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex);
|
|
113291
|
+
sIndex += compSize;
|
|
113263
113292
|
}
|
|
113264
|
-
|
|
113293
|
+
}
|
|
113294
|
+
if (useContentSum) {
|
|
113295
|
+
sIndex += 4;
|
|
113296
|
+
}
|
|
113297
|
+
return dIndex;
|
|
113265
113298
|
}
|
|
113266
|
-
function
|
|
113267
|
-
|
|
113268
|
-
|
|
113269
|
-
|
|
113270
|
-
|
|
113271
|
-
|
|
113272
|
-
|
|
113273
|
-
|
|
113274
|
-
|
|
113275
|
-
|
|
113276
|
-
|
|
113277
|
-
// them (the one with no brace set left in the tail).
|
|
113278
|
-
let dropEmpties = false;
|
|
113279
|
-
let firstGroup = true;
|
|
113280
|
-
for (;;) {
|
|
113281
|
-
const m = balanced('{', '}', str);
|
|
113282
|
-
// No brace set left: the rest of the string is literal.
|
|
113283
|
-
if (!m) {
|
|
113284
|
-
return combine(acc, str, [''], max, maxLength, dropEmpties);
|
|
113285
|
-
}
|
|
113286
|
-
// no need to expand pre, since it is guaranteed to be free of brace-sets
|
|
113287
|
-
const pre = m.pre;
|
|
113288
|
-
if (/\$$/.test(pre)) {
|
|
113289
|
-
acc = combine(acc, pre + '{' + m.body + '}', [''], max, maxLength, dropEmpties && !m.post.length);
|
|
113290
|
-
firstGroup = false;
|
|
113291
|
-
if (!m.post.length)
|
|
113292
|
-
break;
|
|
113293
|
-
str = m.post;
|
|
113294
|
-
continue;
|
|
113295
|
-
}
|
|
113296
|
-
const isNumericSequence = /^-?\d+\.\.-?\d+(?:\.\.-?\d+)?$/.test(m.body);
|
|
113297
|
-
const isAlphaSequence = /^[a-zA-Z]\.\.[a-zA-Z](?:\.\.-?\d+)?$/.test(m.body);
|
|
113298
|
-
const isSequence = isNumericSequence || isAlphaSequence;
|
|
113299
|
-
const isOptions = m.body.indexOf(',') >= 0;
|
|
113300
|
-
if (!isSequence && !isOptions) {
|
|
113301
|
-
// {a},b}
|
|
113302
|
-
if (m.post.match(/,(?!,).*\}/)) {
|
|
113303
|
-
str = m.pre + '{' + m.body + escClose + m.post;
|
|
113304
|
-
isTop = true;
|
|
113305
|
-
continue;
|
|
113306
|
-
}
|
|
113307
|
-
// Nothing here expands, so the whole remaining string is literal.
|
|
113308
|
-
return combine(acc, pre + '{' + m.body + '}' + m.post, [''], max, maxLength, dropEmpties);
|
|
113309
|
-
}
|
|
113310
|
-
if (firstGroup) {
|
|
113311
|
-
dropEmpties = isTop && !isSequence;
|
|
113312
|
-
firstGroup = false;
|
|
113313
|
-
}
|
|
113314
|
-
let values;
|
|
113315
|
-
if (isSequence) {
|
|
113316
|
-
values = expandSequence(m.body, isAlphaSequence, max, maxLength);
|
|
113317
|
-
}
|
|
113318
|
-
else {
|
|
113319
|
-
let n = parseCommaParts(m.body);
|
|
113320
|
-
if (n.length === 1 && n[0] !== undefined) {
|
|
113321
|
-
// x{{a,b}}y ==> x{a}y x{b}y
|
|
113322
|
-
n = expand_(n[0], max, maxLength, false).map(embrace);
|
|
113323
|
-
//XXX is this necessary? Can't seem to hit it in tests.
|
|
113324
|
-
/* c8 ignore start */
|
|
113325
|
-
if (n.length === 1) {
|
|
113326
|
-
acc = combine(acc, pre + n[0], [''], max, maxLength, dropEmpties && !m.post.length);
|
|
113327
|
-
if (!m.post.length)
|
|
113328
|
-
break;
|
|
113329
|
-
str = m.post;
|
|
113330
|
-
continue;
|
|
113331
|
-
}
|
|
113332
|
-
/* c8 ignore stop */
|
|
113333
|
-
}
|
|
113334
|
-
// Values that `combine` is going to drop as empty produce no result, so
|
|
113335
|
-
// they must not count against `max` - otherwise `{a,,b}` with `max: 2`
|
|
113336
|
-
// would stop at `['a', '']` and yield one result instead of two. Skipping
|
|
113337
|
-
// them outright keeps `values` bounded while leaving `max` a bound on
|
|
113338
|
-
// *kept* results.
|
|
113339
|
-
let dropsEmpties = dropEmpties && !m.post.length && !pre;
|
|
113340
|
-
for (let d = 0; dropsEmpties && d < acc.length; d++) {
|
|
113341
|
-
if (acc[d]) {
|
|
113342
|
-
dropsEmpties = false;
|
|
113343
|
-
}
|
|
113344
|
-
}
|
|
113345
|
-
values = [];
|
|
113346
|
-
let valuesLength = 0;
|
|
113347
|
-
outer: for (let j = 0; j < n.length; j++) {
|
|
113348
|
-
const expanded = expand_(n[j], max, maxLength, false);
|
|
113349
|
-
for (let k = 0; k < expanded.length; k++) {
|
|
113350
|
-
const v = expanded[k];
|
|
113351
|
-
if (dropsEmpties && !v)
|
|
113352
|
-
continue;
|
|
113353
|
-
if (values.length >= max || valuesLength + v.length > maxLength) {
|
|
113354
|
-
break outer;
|
|
113355
|
-
}
|
|
113356
|
-
values.push(v);
|
|
113357
|
-
valuesLength += v.length;
|
|
113358
|
-
}
|
|
113359
|
-
}
|
|
113360
|
-
}
|
|
113361
|
-
acc = combine(acc, pre, values, max, maxLength, dropEmpties && !m.post.length);
|
|
113362
|
-
if (!m.post.length)
|
|
113363
|
-
break;
|
|
113364
|
-
str = m.post;
|
|
113365
|
-
}
|
|
113366
|
-
return acc;
|
|
113299
|
+
function decompress(src, maxSize) {
|
|
113300
|
+
let dst, size;
|
|
113301
|
+
if (maxSize === void 0) {
|
|
113302
|
+
maxSize = decompressBound(src);
|
|
113303
|
+
}
|
|
113304
|
+
dst = makeBuffer(maxSize);
|
|
113305
|
+
size = decompressFrame(src, dst);
|
|
113306
|
+
if (size !== maxSize) {
|
|
113307
|
+
dst = sliceArray(dst, 0, size);
|
|
113308
|
+
}
|
|
113309
|
+
return dst;
|
|
113367
113310
|
}
|
|
113368
113311
|
|
|
113369
|
-
|
|
113370
|
-
|
|
113371
|
-
|
|
113372
|
-
|
|
113312
|
+
// src/utils/RangeList.ts
|
|
113313
|
+
var RangeList = class {
|
|
113314
|
+
ranges = [];
|
|
113315
|
+
/**
|
|
113316
|
+
* Add a range to the list. If it overlaps with existing ranges,
|
|
113317
|
+
* it will split them and increment reference counts accordingly.
|
|
113318
|
+
*/
|
|
113319
|
+
add(start, end) {
|
|
113320
|
+
if (end <= start) {
|
|
113321
|
+
throw new TypeError("End must be greater than start");
|
|
113373
113322
|
}
|
|
113374
|
-
|
|
113375
|
-
|
|
113323
|
+
const overlappingRanges = [];
|
|
113324
|
+
for (let i = 0; i < this.ranges.length; i++) {
|
|
113325
|
+
const range2 = this.ranges[i];
|
|
113326
|
+
if (start < range2.end && end > range2.start) {
|
|
113327
|
+
overlappingRanges.push({ index: i, range: range2 });
|
|
113328
|
+
}
|
|
113329
|
+
if (range2.data !== null) {
|
|
113330
|
+
throw new Error("Overlapping range already has data");
|
|
113331
|
+
}
|
|
113376
113332
|
}
|
|
113377
|
-
|
|
113378
|
-
|
|
113379
|
-
|
|
113380
|
-
|
|
113381
|
-
// { <posix class>: [<translation>, /u flag required, negated]
|
|
113382
|
-
const posixClasses = {
|
|
113383
|
-
'[:alnum:]': ['\\p{L}\\p{Nl}\\p{Nd}', true],
|
|
113384
|
-
'[:alpha:]': ['\\p{L}\\p{Nl}', true],
|
|
113385
|
-
'[:ascii:]': ['\\x' + '00-\\x' + '7f', false],
|
|
113386
|
-
'[:blank:]': ['\\p{Zs}\\t', true],
|
|
113387
|
-
'[:cntrl:]': ['\\p{Cc}', true],
|
|
113388
|
-
'[:digit:]': ['\\p{Nd}', true],
|
|
113389
|
-
'[:graph:]': ['\\p{Z}\\p{C}', true, true],
|
|
113390
|
-
'[:lower:]': ['\\p{Ll}', true],
|
|
113391
|
-
'[:print:]': ['\\p{C}', true],
|
|
113392
|
-
'[:punct:]': ['\\p{P}', true],
|
|
113393
|
-
'[:space:]': ['\\p{Z}\\t\\r\\n\\v\\f', true],
|
|
113394
|
-
'[:upper:]': ['\\p{Lu}', true],
|
|
113395
|
-
'[:word:]': ['\\p{L}\\p{Nl}\\p{Nd}\\p{Pc}', true],
|
|
113396
|
-
'[:xdigit:]': ['A-Fa-f0-9', false],
|
|
113397
|
-
};
|
|
113398
|
-
// only need to escape a few things inside of brace expressions
|
|
113399
|
-
// escapes: [ \ ] -
|
|
113400
|
-
const braceEscape = (s) => s.replace(/[[\]\\-]/g, '\\$&');
|
|
113401
|
-
// escape all regexp magic characters
|
|
113402
|
-
const regexpEscape = (s) => s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&');
|
|
113403
|
-
// everything has already been escaped, we just have to join
|
|
113404
|
-
const rangesToString = (ranges) => ranges.join('');
|
|
113405
|
-
// takes a glob string at a posix brace expression, and returns
|
|
113406
|
-
// an equivalent regular expression source, and boolean indicating
|
|
113407
|
-
// whether the /u flag needs to be applied, and the number of chars
|
|
113408
|
-
// consumed to parse the character class.
|
|
113409
|
-
// This also removes out of order ranges, and returns ($.) if the
|
|
113410
|
-
// entire class just no good.
|
|
113411
|
-
const parseClass = (glob, position) => {
|
|
113412
|
-
const pos = position;
|
|
113413
|
-
/* c8 ignore start */
|
|
113414
|
-
if (glob.charAt(pos) !== '[') {
|
|
113415
|
-
throw new Error('not in a brace expression');
|
|
113333
|
+
if (overlappingRanges.length === 0) {
|
|
113334
|
+
this.ranges.push({ start, end, refCount: 1, data: null });
|
|
113335
|
+
this.ranges.sort((a, b) => a.start - b.start);
|
|
113336
|
+
return;
|
|
113416
113337
|
}
|
|
113417
|
-
|
|
113418
|
-
|
|
113419
|
-
|
|
113420
|
-
|
|
113421
|
-
|
|
113422
|
-
|
|
113423
|
-
|
|
113424
|
-
|
|
113425
|
-
|
|
113426
|
-
|
|
113427
|
-
|
|
113428
|
-
|
|
113429
|
-
|
|
113430
|
-
|
|
113431
|
-
|
|
113432
|
-
|
|
113433
|
-
|
|
113434
|
-
|
|
113435
|
-
|
|
113436
|
-
|
|
113437
|
-
|
|
113438
|
-
|
|
113439
|
-
|
|
113440
|
-
|
|
113441
|
-
|
|
113442
|
-
|
|
113443
|
-
|
|
113444
|
-
|
|
113445
|
-
|
|
113446
|
-
|
|
113447
|
-
|
|
113448
|
-
|
|
113449
|
-
|
|
113450
|
-
|
|
113451
|
-
// invalid, [a-[] is fine, but not [a-[:alpha]]
|
|
113452
|
-
if (rangeStart) {
|
|
113453
|
-
return ['$.', false, glob.length - pos, true];
|
|
113454
|
-
}
|
|
113455
|
-
i += cls.length;
|
|
113456
|
-
if (neg)
|
|
113457
|
-
negs.push(unip);
|
|
113458
|
-
else
|
|
113459
|
-
ranges.push(unip);
|
|
113460
|
-
uflag = uflag || u;
|
|
113461
|
-
continue WHILE;
|
|
113462
|
-
}
|
|
113463
|
-
}
|
|
113464
|
-
}
|
|
113465
|
-
// now it's just a normal character, effectively
|
|
113466
|
-
escaping = false;
|
|
113467
|
-
if (rangeStart) {
|
|
113468
|
-
// throw this range away if it's not valid, but others
|
|
113469
|
-
// can still match.
|
|
113470
|
-
if (c > rangeStart) {
|
|
113471
|
-
ranges.push(braceEscape(rangeStart) + '-' + braceEscape(c));
|
|
113472
|
-
}
|
|
113473
|
-
else if (c === rangeStart) {
|
|
113474
|
-
ranges.push(braceEscape(c));
|
|
113475
|
-
}
|
|
113476
|
-
rangeStart = '';
|
|
113477
|
-
i++;
|
|
113478
|
-
continue;
|
|
113479
|
-
}
|
|
113480
|
-
// now might be the start of a range.
|
|
113481
|
-
// can be either c-d or c-] or c<more...>] or c] at this point
|
|
113482
|
-
if (glob.startsWith('-]', i + 1)) {
|
|
113483
|
-
ranges.push(braceEscape(c + '-'));
|
|
113484
|
-
i += 2;
|
|
113485
|
-
continue;
|
|
113486
|
-
}
|
|
113487
|
-
if (glob.startsWith('-', i + 1)) {
|
|
113488
|
-
rangeStart = c;
|
|
113489
|
-
i += 2;
|
|
113490
|
-
continue;
|
|
113491
|
-
}
|
|
113492
|
-
// not the start of a range, just a single character
|
|
113493
|
-
ranges.push(braceEscape(c));
|
|
113494
|
-
i++;
|
|
113338
|
+
const newRanges = [];
|
|
113339
|
+
let currentPos = start;
|
|
113340
|
+
for (let i = 0; i < overlappingRanges.length; i++) {
|
|
113341
|
+
const { range: range2 } = overlappingRanges[i];
|
|
113342
|
+
if (currentPos < range2.start) {
|
|
113343
|
+
newRanges.push({
|
|
113344
|
+
start: currentPos,
|
|
113345
|
+
end: range2.start,
|
|
113346
|
+
refCount: 1,
|
|
113347
|
+
data: null
|
|
113348
|
+
});
|
|
113349
|
+
} else if (range2.start < currentPos) {
|
|
113350
|
+
newRanges.push({
|
|
113351
|
+
start: range2.start,
|
|
113352
|
+
end: currentPos,
|
|
113353
|
+
refCount: range2.refCount,
|
|
113354
|
+
data: null
|
|
113355
|
+
});
|
|
113356
|
+
}
|
|
113357
|
+
newRanges.push({
|
|
113358
|
+
start: Math.max(currentPos, range2.start),
|
|
113359
|
+
end: Math.min(end, range2.end),
|
|
113360
|
+
refCount: range2.refCount + 1,
|
|
113361
|
+
data: null
|
|
113362
|
+
});
|
|
113363
|
+
if (range2.end > end) {
|
|
113364
|
+
newRanges.push({
|
|
113365
|
+
start: end,
|
|
113366
|
+
end: range2.end,
|
|
113367
|
+
refCount: range2.refCount,
|
|
113368
|
+
data: null
|
|
113369
|
+
});
|
|
113370
|
+
}
|
|
113371
|
+
currentPos = Math.max(currentPos, range2.end);
|
|
113495
113372
|
}
|
|
113496
|
-
if (
|
|
113497
|
-
|
|
113498
|
-
|
|
113499
|
-
|
|
113373
|
+
if (currentPos < end) {
|
|
113374
|
+
newRanges.push({
|
|
113375
|
+
start: currentPos,
|
|
113376
|
+
end,
|
|
113377
|
+
refCount: 1,
|
|
113378
|
+
data: null
|
|
113379
|
+
});
|
|
113500
113380
|
}
|
|
113501
|
-
|
|
113502
|
-
|
|
113503
|
-
|
|
113504
|
-
|
|
113381
|
+
const firstIndex = overlappingRanges[0].index;
|
|
113382
|
+
const lastIndex = overlappingRanges[overlappingRanges.length - 1].index;
|
|
113383
|
+
this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges);
|
|
113384
|
+
this.ranges.sort((a, b) => a.start - b.start);
|
|
113385
|
+
}
|
|
113386
|
+
/**
|
|
113387
|
+
* Remove a range from the list. The range must start and end at existing boundaries.
|
|
113388
|
+
*/
|
|
113389
|
+
remove(start, end) {
|
|
113390
|
+
if (end <= start) {
|
|
113391
|
+
throw new TypeError("End must be greater than start");
|
|
113505
113392
|
}
|
|
113506
|
-
|
|
113507
|
-
|
|
113508
|
-
|
|
113509
|
-
|
|
113510
|
-
|
|
113511
|
-
|
|
113512
|
-
/^\\?.$/.test(ranges[0]) &&
|
|
113513
|
-
!negate) {
|
|
113514
|
-
const r = ranges[0].length === 2 ? ranges[0].slice(-1) : ranges[0];
|
|
113515
|
-
return [regexpEscape(r), false, endPos - pos, false];
|
|
113393
|
+
const affectedRanges = [];
|
|
113394
|
+
for (let i = 0; i < this.ranges.length; i++) {
|
|
113395
|
+
const range2 = this.ranges[i];
|
|
113396
|
+
if (start < range2.end && end > range2.start) {
|
|
113397
|
+
affectedRanges.push({ index: i, range: range2 });
|
|
113398
|
+
}
|
|
113516
113399
|
}
|
|
113517
|
-
|
|
113518
|
-
|
|
113519
|
-
const comb = ranges.length && negs.length ? '(' + sranges + '|' + snegs + ')'
|
|
113520
|
-
: ranges.length ? sranges
|
|
113521
|
-
: snegs;
|
|
113522
|
-
return [comb, uflag, endPos - pos, true];
|
|
113523
|
-
};
|
|
113524
|
-
|
|
113525
|
-
/**
|
|
113526
|
-
* Un-escape a string that has been escaped with {@link escape}.
|
|
113527
|
-
*
|
|
113528
|
-
* If the {@link MinimatchOptions.windowsPathsNoEscape} option is used, then
|
|
113529
|
-
* square-bracket escapes are removed, but not backslash escapes.
|
|
113530
|
-
*
|
|
113531
|
-
* For example, it will turn the string `'[*]'` into `*`, but it will not
|
|
113532
|
-
* turn `'\\*'` into `'*'`, because `\` is a path separator in
|
|
113533
|
-
* `windowsPathsNoEscape` mode.
|
|
113534
|
-
*
|
|
113535
|
-
* When `windowsPathsNoEscape` is not set, then both square-bracket escapes and
|
|
113536
|
-
* backslash escapes are removed.
|
|
113537
|
-
*
|
|
113538
|
-
* Slashes (and backslashes in `windowsPathsNoEscape` mode) cannot be escaped
|
|
113539
|
-
* or unescaped.
|
|
113540
|
-
*
|
|
113541
|
-
* When `magicalBraces` is not set, escapes of braces (`{` and `}`) will not be
|
|
113542
|
-
* unescaped.
|
|
113543
|
-
*/
|
|
113544
|
-
const unescape$1 = (s, { windowsPathsNoEscape = false, magicalBraces = true, } = {}) => {
|
|
113545
|
-
if (magicalBraces) {
|
|
113546
|
-
return windowsPathsNoEscape ?
|
|
113547
|
-
s.replace(/\[([^/\\])\]/g, '$1')
|
|
113548
|
-
: s
|
|
113549
|
-
.replace(/((?!\\).|^)\[([^/\\])\]/g, '$1$2')
|
|
113550
|
-
.replace(/\\([^/])/g, '$1');
|
|
113400
|
+
if (affectedRanges.length === 0) {
|
|
113401
|
+
throw new Error("No ranges found to remove");
|
|
113551
113402
|
}
|
|
113552
|
-
|
|
113553
|
-
|
|
113554
|
-
|
|
113555
|
-
|
|
113556
|
-
|
|
113403
|
+
if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) {
|
|
113404
|
+
throw new Error("Range boundaries must match existing boundaries");
|
|
113405
|
+
}
|
|
113406
|
+
for (let i = 0; i < affectedRanges.length; i++) {
|
|
113407
|
+
const { range: range2 } = affectedRanges[i];
|
|
113408
|
+
range2.refCount--;
|
|
113409
|
+
}
|
|
113410
|
+
this.ranges = this.ranges.filter((range2) => range2.refCount > 0);
|
|
113411
|
+
}
|
|
113412
|
+
/**
|
|
113413
|
+
* Get all ranges within the specified boundaries.
|
|
113414
|
+
*/
|
|
113415
|
+
getRanges(start, end) {
|
|
113416
|
+
if (end <= start) {
|
|
113417
|
+
throw new TypeError("End must be greater than start");
|
|
113418
|
+
}
|
|
113419
|
+
return this.ranges.filter((range2) => start < range2.end && end > range2.start);
|
|
113420
|
+
}
|
|
113421
|
+
/**
|
|
113422
|
+
* Get all ranges in the list
|
|
113423
|
+
*/
|
|
113424
|
+
getAllRanges() {
|
|
113425
|
+
return [...this.ranges];
|
|
113426
|
+
}
|
|
113557
113427
|
};
|
|
113558
113428
|
|
|
113559
|
-
//
|
|
113560
|
-
var
|
|
113561
|
-
|
|
113562
|
-
|
|
113563
|
-
|
|
113564
|
-
|
|
113565
|
-
|
|
113566
|
-
|
|
113567
|
-
|
|
113568
|
-
|
|
113569
|
-
|
|
113570
|
-
|
|
113571
|
-
|
|
113572
|
-
|
|
113573
|
-
|
|
113574
|
-
|
|
113575
|
-
|
|
113576
|
-
|
|
113577
|
-
|
|
113578
|
-
|
|
113579
|
-
|
|
113580
|
-
|
|
113581
|
-
|
|
113582
|
-
|
|
113583
|
-
|
|
113584
|
-
|
|
113585
|
-
|
|
113586
|
-
|
|
113587
|
-
|
|
113588
|
-
|
|
113589
|
-
|
|
113590
|
-
|
|
113591
|
-
|
|
113592
|
-
|
|
113593
|
-
|
|
113594
|
-
|
|
113595
|
-
|
|
113596
|
-
|
|
113429
|
+
// src/utils/XetBlob.ts
|
|
113430
|
+
var JWT_SAFETY_PERIOD = 6e4;
|
|
113431
|
+
var JWT_CACHE_SIZE = 1e3;
|
|
113432
|
+
var compressionSchemeLabels = {
|
|
113433
|
+
[0 /* None */]: "None",
|
|
113434
|
+
[1 /* LZ4 */]: "LZ4",
|
|
113435
|
+
[2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4"
|
|
113436
|
+
};
|
|
113437
|
+
var XET_CHUNK_HEADER_BYTES = 8;
|
|
113438
|
+
var XetBlob = class extends Blob {
|
|
113439
|
+
fetch;
|
|
113440
|
+
accessToken;
|
|
113441
|
+
refreshUrl;
|
|
113442
|
+
reconstructionUrl;
|
|
113443
|
+
hash;
|
|
113444
|
+
start = 0;
|
|
113445
|
+
end = 0;
|
|
113446
|
+
internalLogging = false;
|
|
113447
|
+
reconstructionInfo;
|
|
113448
|
+
listener;
|
|
113449
|
+
constructor(params) {
|
|
113450
|
+
super([]);
|
|
113451
|
+
this.fetch = params.fetch ?? fetch.bind(globalThis);
|
|
113452
|
+
this.accessToken = checkCredentials(params);
|
|
113453
|
+
this.refreshUrl = params.refreshUrl;
|
|
113454
|
+
this.end = params.size;
|
|
113455
|
+
this.reconstructionUrl = params.reconstructionUrl;
|
|
113456
|
+
this.hash = params.hash;
|
|
113457
|
+
this.listener = params.listener;
|
|
113458
|
+
this.internalLogging = params.internalLogging ?? false;
|
|
113459
|
+
this.refreshUrl;
|
|
113460
|
+
}
|
|
113461
|
+
get size() {
|
|
113462
|
+
return this.end - this.start;
|
|
113463
|
+
}
|
|
113464
|
+
#clone() {
|
|
113465
|
+
const blob = new XetBlob({
|
|
113466
|
+
fetch: this.fetch,
|
|
113467
|
+
hash: this.hash,
|
|
113468
|
+
refreshUrl: this.refreshUrl,
|
|
113469
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
113470
|
+
reconstructionUrl: this.reconstructionUrl,
|
|
113471
|
+
size: this.size
|
|
113472
|
+
});
|
|
113473
|
+
blob.accessToken = this.accessToken;
|
|
113474
|
+
blob.start = this.start;
|
|
113475
|
+
blob.end = this.end;
|
|
113476
|
+
blob.reconstructionInfo = this.reconstructionInfo;
|
|
113477
|
+
blob.listener = this.listener;
|
|
113478
|
+
blob.internalLogging = this.internalLogging;
|
|
113479
|
+
return blob;
|
|
113480
|
+
}
|
|
113481
|
+
slice(start = 0, end = this.size) {
|
|
113482
|
+
const slice = this.#clone();
|
|
113483
|
+
slice.start = this.start + start;
|
|
113484
|
+
slice.end = Math.min(this.start + end, this.end);
|
|
113485
|
+
if (slice.start !== this.start || slice.end !== this.end) {
|
|
113486
|
+
slice.reconstructionInfo = void 0;
|
|
113487
|
+
}
|
|
113488
|
+
return slice;
|
|
113489
|
+
}
|
|
113490
|
+
#reconstructionInfoPromise;
|
|
113491
|
+
#loadReconstructionInfo() {
|
|
113492
|
+
if (this.#reconstructionInfoPromise) {
|
|
113493
|
+
return this.#reconstructionInfoPromise;
|
|
113494
|
+
}
|
|
113495
|
+
this.#reconstructionInfoPromise = (async () => {
|
|
113496
|
+
const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl);
|
|
113497
|
+
const resp = await this.fetch(this.reconstructionUrl ?? `${connParams.casUrl}/v1/reconstructions/${this.hash}`, {
|
|
113498
|
+
headers: {
|
|
113499
|
+
Authorization: `Bearer ${connParams.accessToken}`,
|
|
113500
|
+
Range: `bytes=${this.start}-${this.end - 1}`
|
|
113501
|
+
}
|
|
113502
|
+
});
|
|
113503
|
+
if (!resp.ok) {
|
|
113504
|
+
throw await createApiError(resp);
|
|
113505
|
+
}
|
|
113506
|
+
this.reconstructionInfo = await resp.json();
|
|
113507
|
+
return this.reconstructionInfo;
|
|
113508
|
+
})().finally(() => this.#reconstructionInfoPromise = void 0);
|
|
113509
|
+
return this.#reconstructionInfoPromise;
|
|
113510
|
+
}
|
|
113511
|
+
async #fetch() {
|
|
113512
|
+
if (!this.reconstructionInfo) {
|
|
113513
|
+
await this.#loadReconstructionInfo();
|
|
113514
|
+
}
|
|
113515
|
+
const rangeLists = /* @__PURE__ */ new Map();
|
|
113516
|
+
if (!this.reconstructionInfo) {
|
|
113517
|
+
throw new Error("Failed to load reconstruction info");
|
|
113518
|
+
}
|
|
113519
|
+
for (const term of this.reconstructionInfo.terms) {
|
|
113520
|
+
let rangeList = rangeLists.get(term.hash);
|
|
113521
|
+
if (!rangeList) {
|
|
113522
|
+
rangeList = new RangeList();
|
|
113523
|
+
rangeLists.set(term.hash, rangeList);
|
|
113524
|
+
}
|
|
113525
|
+
rangeList.add(term.range.start, term.range.end);
|
|
113526
|
+
}
|
|
113527
|
+
const listener = this.listener;
|
|
113528
|
+
const log = this.internalLogging ? (...args) => console.log(...args) : () => {
|
|
113529
|
+
};
|
|
113530
|
+
async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) {
|
|
113531
|
+
let totalBytesRead = 0;
|
|
113532
|
+
let readBytesToSkip = reconstructionInfo.offset_into_first_range;
|
|
113533
|
+
for (const term of reconstructionInfo.terms) {
|
|
113534
|
+
if (totalBytesRead >= maxBytes) {
|
|
113535
|
+
break;
|
|
113536
|
+
}
|
|
113537
|
+
const rangeList = rangeLists.get(term.hash);
|
|
113538
|
+
if (!rangeList) {
|
|
113539
|
+
throw new Error(`Failed to find range list for term ${term.hash}`);
|
|
113540
|
+
}
|
|
113541
|
+
{
|
|
113542
|
+
const termRanges = rangeList.getRanges(term.range.start, term.range.end);
|
|
113543
|
+
if (termRanges.every((range2) => range2.data)) {
|
|
113544
|
+
log("all data available for term", term.hash, readBytesToSkip);
|
|
113545
|
+
rangeLoop:
|
|
113546
|
+
for (const range2 of termRanges) {
|
|
113547
|
+
for (let chunk2 of range2.data) {
|
|
113548
|
+
if (readBytesToSkip) {
|
|
113549
|
+
const skipped = Math.min(readBytesToSkip, chunk2.byteLength);
|
|
113550
|
+
chunk2 = chunk2.slice(skipped);
|
|
113551
|
+
readBytesToSkip -= skipped;
|
|
113552
|
+
if (!chunk2.byteLength) {
|
|
113553
|
+
continue;
|
|
113554
|
+
}
|
|
113555
|
+
}
|
|
113556
|
+
if (chunk2.byteLength > maxBytes - totalBytesRead) {
|
|
113557
|
+
chunk2 = chunk2.slice(0, maxBytes - totalBytesRead);
|
|
113558
|
+
}
|
|
113559
|
+
totalBytesRead += chunk2.byteLength;
|
|
113560
|
+
yield range2.refCount > 1 ? chunk2.slice() : chunk2;
|
|
113561
|
+
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
113562
|
+
if (totalBytesRead >= maxBytes) {
|
|
113563
|
+
break rangeLoop;
|
|
113564
|
+
}
|
|
113565
|
+
}
|
|
113566
|
+
}
|
|
113567
|
+
rangeList.remove(term.range.start, term.range.end);
|
|
113568
|
+
continue;
|
|
113569
|
+
}
|
|
113570
|
+
}
|
|
113571
|
+
const fetchInfo = reconstructionInfo.fetch_info[term.hash].find(
|
|
113572
|
+
(info) => info.range.start <= term.range.start && info.range.end >= term.range.end
|
|
113573
|
+
);
|
|
113574
|
+
if (!fetchInfo) {
|
|
113575
|
+
throw new Error(
|
|
113576
|
+
`Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}`
|
|
113577
|
+
);
|
|
113578
|
+
}
|
|
113579
|
+
log("term", term);
|
|
113580
|
+
log("fetchinfo", fetchInfo);
|
|
113581
|
+
log("readBytesToSkip", readBytesToSkip);
|
|
113582
|
+
let resp = await customFetch(fetchInfo.url, {
|
|
113583
|
+
headers: {
|
|
113584
|
+
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
113585
|
+
}
|
|
113586
|
+
});
|
|
113587
|
+
if (resp.status === 403) {
|
|
113588
|
+
reconstructionInfo = await reloadReconstructionInfo();
|
|
113589
|
+
resp = await customFetch(fetchInfo.url, {
|
|
113590
|
+
headers: {
|
|
113591
|
+
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
113592
|
+
}
|
|
113593
|
+
});
|
|
113594
|
+
}
|
|
113595
|
+
if (!resp.ok) {
|
|
113596
|
+
throw await createApiError(resp);
|
|
113597
|
+
}
|
|
113598
|
+
log(
|
|
113599
|
+
"expected content length",
|
|
113600
|
+
resp.headers.get("content-length"),
|
|
113601
|
+
"range",
|
|
113602
|
+
fetchInfo.url_range,
|
|
113603
|
+
resp.headers.get("content-range")
|
|
113604
|
+
);
|
|
113605
|
+
const reader = resp.body?.getReader();
|
|
113606
|
+
if (!reader) {
|
|
113607
|
+
throw new Error("Failed to get reader from response body");
|
|
113608
|
+
}
|
|
113609
|
+
let done = false;
|
|
113610
|
+
let chunkIndex = fetchInfo.range.start;
|
|
113611
|
+
const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end);
|
|
113612
|
+
let leftoverBytes = void 0;
|
|
113613
|
+
let totalFetchBytes = 0;
|
|
113614
|
+
fetchData:
|
|
113615
|
+
while (!done && totalBytesRead < maxBytes) {
|
|
113616
|
+
const result = await reader.read();
|
|
113617
|
+
listener?.({ event: "read" });
|
|
113618
|
+
done = result.done;
|
|
113619
|
+
log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip);
|
|
113620
|
+
if (!result.value) {
|
|
113621
|
+
log("no data in result, cancelled", result);
|
|
113622
|
+
continue;
|
|
113623
|
+
}
|
|
113624
|
+
totalFetchBytes += result.value.byteLength;
|
|
113625
|
+
if (leftoverBytes) {
|
|
113626
|
+
result.value = combineUint8Arrays(leftoverBytes, result.value);
|
|
113627
|
+
leftoverBytes = void 0;
|
|
113628
|
+
}
|
|
113629
|
+
while (totalBytesRead < maxBytes && result.value?.byteLength) {
|
|
113630
|
+
if (result.value.byteLength < 8) {
|
|
113631
|
+
leftoverBytes = result.value;
|
|
113632
|
+
continue fetchData;
|
|
113633
|
+
}
|
|
113634
|
+
const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES);
|
|
113635
|
+
const chunkHeader = {
|
|
113636
|
+
version: header.getUint8(0),
|
|
113637
|
+
compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16,
|
|
113638
|
+
compression_scheme: header.getUint8(4),
|
|
113639
|
+
uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16
|
|
113640
|
+
};
|
|
113641
|
+
log("chunk header", chunkHeader, "to skip", readBytesToSkip);
|
|
113642
|
+
if (chunkHeader.version !== 0) {
|
|
113643
|
+
throw new Error(`Unsupported chunk version ${chunkHeader.version}`);
|
|
113644
|
+
}
|
|
113645
|
+
if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) {
|
|
113646
|
+
throw new Error(
|
|
113647
|
+
`Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}`
|
|
113648
|
+
);
|
|
113649
|
+
}
|
|
113650
|
+
if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) {
|
|
113651
|
+
leftoverBytes = result.value;
|
|
113652
|
+
continue fetchData;
|
|
113653
|
+
}
|
|
113654
|
+
result.value = result.value.slice(XET_CHUNK_HEADER_BYTES);
|
|
113655
|
+
let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes(
|
|
113656
|
+
decompress(
|
|
113657
|
+
result.value.slice(0, chunkHeader.compressed_length),
|
|
113658
|
+
chunkHeader.uncompressed_length
|
|
113659
|
+
)
|
|
113660
|
+
) : result.value.slice(0, chunkHeader.compressed_length);
|
|
113661
|
+
const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end);
|
|
113662
|
+
const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end;
|
|
113663
|
+
const minRefCountToStore = shouldYield ? 2 : 1;
|
|
113664
|
+
let stored = false;
|
|
113665
|
+
if (range2 && range2.refCount >= minRefCountToStore) {
|
|
113666
|
+
range2.data ??= [];
|
|
113667
|
+
range2.data.push(uncompressed);
|
|
113668
|
+
stored = true;
|
|
113669
|
+
}
|
|
113670
|
+
if (shouldYield) {
|
|
113671
|
+
if (readBytesToSkip) {
|
|
113672
|
+
const skipped = Math.min(readBytesToSkip, uncompressed.byteLength);
|
|
113673
|
+
uncompressed = uncompressed.slice(readBytesToSkip);
|
|
113674
|
+
readBytesToSkip -= skipped;
|
|
113675
|
+
}
|
|
113676
|
+
if (uncompressed.byteLength > maxBytes - totalBytesRead) {
|
|
113677
|
+
uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead);
|
|
113678
|
+
}
|
|
113679
|
+
if (uncompressed.byteLength) {
|
|
113680
|
+
log(
|
|
113681
|
+
"yield",
|
|
113682
|
+
uncompressed.byteLength,
|
|
113683
|
+
"bytes",
|
|
113684
|
+
result.value.byteLength,
|
|
113685
|
+
"total read",
|
|
113686
|
+
totalBytesRead,
|
|
113687
|
+
stored
|
|
113688
|
+
);
|
|
113689
|
+
totalBytesRead += uncompressed.byteLength;
|
|
113690
|
+
yield stored ? uncompressed.slice() : uncompressed;
|
|
113691
|
+
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
113692
|
+
}
|
|
113693
|
+
}
|
|
113694
|
+
chunkIndex++;
|
|
113695
|
+
result.value = result.value.slice(chunkHeader.compressed_length);
|
|
113696
|
+
}
|
|
113697
|
+
}
|
|
113698
|
+
if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) {
|
|
113699
|
+
log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes);
|
|
113700
|
+
log("failed to fetch all data for term", term.hash);
|
|
113701
|
+
throw new Error(
|
|
113702
|
+
`Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}`
|
|
113703
|
+
);
|
|
113704
|
+
}
|
|
113705
|
+
log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes);
|
|
113706
|
+
log("cancel reader");
|
|
113707
|
+
await reader.cancel();
|
|
113708
|
+
}
|
|
113709
|
+
}
|
|
113710
|
+
const iterator = readData(
|
|
113711
|
+
this.reconstructionInfo,
|
|
113712
|
+
this.fetch,
|
|
113713
|
+
this.end - this.start,
|
|
113714
|
+
this.#loadReconstructionInfo.bind(this)
|
|
113715
|
+
);
|
|
113716
|
+
return new ReadableStream(
|
|
113717
|
+
{
|
|
113718
|
+
// todo: when Safari supports it, type controller as ReadableByteStreamController
|
|
113719
|
+
async pull(controller) {
|
|
113720
|
+
const result = await iterator.next();
|
|
113721
|
+
if (result.value) {
|
|
113722
|
+
controller.enqueue(result.value);
|
|
113723
|
+
}
|
|
113724
|
+
if (result.done) {
|
|
113725
|
+
controller.close();
|
|
113726
|
+
}
|
|
113727
|
+
},
|
|
113728
|
+
type: "bytes"
|
|
113729
|
+
// todo: when Safari supports it, add autoAllocateChunkSize param
|
|
113730
|
+
},
|
|
113731
|
+
// todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function
|
|
113732
|
+
{
|
|
113733
|
+
highWaterMark: 1e3
|
|
113734
|
+
// 1_000 chunks for ~1MB of RAM
|
|
113735
|
+
}
|
|
113736
|
+
);
|
|
113737
|
+
}
|
|
113738
|
+
async arrayBuffer() {
|
|
113739
|
+
const result = await this.#fetch();
|
|
113740
|
+
return new Response(result).arrayBuffer();
|
|
113741
|
+
}
|
|
113742
|
+
async text() {
|
|
113743
|
+
const result = await this.#fetch();
|
|
113744
|
+
return new Response(result).text();
|
|
113745
|
+
}
|
|
113746
|
+
async response() {
|
|
113747
|
+
const result = await this.#fetch();
|
|
113748
|
+
return new Response(result);
|
|
113749
|
+
}
|
|
113750
|
+
stream() {
|
|
113751
|
+
const stream = new TransformStream();
|
|
113752
|
+
this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
113753
|
+
return stream.readable;
|
|
113754
|
+
}
|
|
113755
|
+
};
|
|
113756
|
+
var jwtPromises = /* @__PURE__ */ new Map();
|
|
113757
|
+
var jwts = /* @__PURE__ */ new Map();
|
|
113758
|
+
function cacheKey(params) {
|
|
113759
|
+
return JSON.stringify([params.refreshUrl, params.initialAccessToken]);
|
|
113760
|
+
}
|
|
113761
|
+
function bg4_regroup_bytes(bytes) {
|
|
113762
|
+
const split = Math.floor(bytes.byteLength / 4);
|
|
113763
|
+
const rem = bytes.byteLength % 4;
|
|
113764
|
+
const g1_pos = split + (rem >= 1 ? 1 : 0);
|
|
113765
|
+
const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0);
|
|
113766
|
+
const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0);
|
|
113767
|
+
const ret = new Uint8Array(bytes.byteLength);
|
|
113768
|
+
for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) {
|
|
113769
|
+
ret[i] = bytes[j];
|
|
113770
|
+
}
|
|
113771
|
+
for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) {
|
|
113772
|
+
ret[i] = bytes[j];
|
|
113773
|
+
}
|
|
113774
|
+
for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) {
|
|
113775
|
+
ret[i] = bytes[j];
|
|
113776
|
+
}
|
|
113777
|
+
for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) {
|
|
113778
|
+
ret[i] = bytes[j];
|
|
113779
|
+
}
|
|
113780
|
+
return ret;
|
|
113781
|
+
}
|
|
113782
|
+
async function getAccessToken(initialAccessToken, customFetch, refreshUrl) {
|
|
113783
|
+
const key = cacheKey({ refreshUrl, initialAccessToken });
|
|
113784
|
+
const jwt = jwts.get(key);
|
|
113785
|
+
if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) {
|
|
113786
|
+
return { accessToken: jwt.accessToken, casUrl: jwt.casUrl };
|
|
113787
|
+
}
|
|
113788
|
+
const existingPromise = jwtPromises.get(key);
|
|
113789
|
+
if (existingPromise) {
|
|
113790
|
+
return existingPromise;
|
|
113791
|
+
}
|
|
113792
|
+
const promise = (async () => {
|
|
113793
|
+
const resp = await customFetch(refreshUrl, {
|
|
113794
|
+
headers: {
|
|
113795
|
+
...initialAccessToken ? {
|
|
113796
|
+
Authorization: `Bearer ${initialAccessToken}`
|
|
113797
|
+
} : {}
|
|
113798
|
+
}
|
|
113799
|
+
});
|
|
113800
|
+
if (!resp.ok) {
|
|
113801
|
+
throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`);
|
|
113802
|
+
}
|
|
113803
|
+
const json = await resp.json();
|
|
113804
|
+
const jwt2 = {
|
|
113805
|
+
accessToken: json.accessToken,
|
|
113806
|
+
expiresAt: new Date(json.exp * 1e3),
|
|
113807
|
+
casUrl: json.casUrl
|
|
113808
|
+
};
|
|
113809
|
+
jwtPromises.delete(key);
|
|
113810
|
+
for (const [key2, value] of jwts.entries()) {
|
|
113811
|
+
if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) {
|
|
113812
|
+
jwts.delete(key2);
|
|
113813
|
+
} else {
|
|
113814
|
+
break;
|
|
113815
|
+
}
|
|
113816
|
+
}
|
|
113817
|
+
if (jwts.size >= JWT_CACHE_SIZE) {
|
|
113818
|
+
const keyToDelete = jwts.keys().next().value;
|
|
113819
|
+
if (keyToDelete) {
|
|
113820
|
+
jwts.delete(keyToDelete);
|
|
113821
|
+
}
|
|
113822
|
+
}
|
|
113823
|
+
jwts.set(key, jwt2);
|
|
113824
|
+
return {
|
|
113825
|
+
accessToken: json.accessToken,
|
|
113826
|
+
casUrl: json.casUrl
|
|
113827
|
+
};
|
|
113828
|
+
})();
|
|
113829
|
+
jwtPromises.set(key, promise);
|
|
113830
|
+
return promise;
|
|
113831
|
+
}
|
|
113832
|
+
|
|
113833
|
+
// src/utils/WebBlob.ts
|
|
113834
|
+
var WebBlob = class extends Blob {
|
|
113835
|
+
static async create(url, opts) {
|
|
113836
|
+
const customFetch = opts?.fetch ?? fetch;
|
|
113837
|
+
const response = await customFetch(url, {
|
|
113838
|
+
method: "HEAD",
|
|
113839
|
+
...opts?.accessToken && {
|
|
113840
|
+
headers: {
|
|
113841
|
+
Authorization: `Bearer ${opts.accessToken}`
|
|
113842
|
+
}
|
|
113843
|
+
}
|
|
113844
|
+
});
|
|
113845
|
+
const size = Number(response.headers.get("content-length"));
|
|
113846
|
+
const contentType = response.headers.get("content-type") || "";
|
|
113847
|
+
const supportRange = response.headers.get("accept-ranges") === "bytes";
|
|
113848
|
+
if (!supportRange || size < (opts?.cacheBelow ?? 1e6)) {
|
|
113849
|
+
return await (await customFetch(url)).blob();
|
|
113850
|
+
}
|
|
113851
|
+
return new WebBlob(url, 0, size, contentType, true, customFetch, opts?.accessToken);
|
|
113852
|
+
}
|
|
113853
|
+
url;
|
|
113854
|
+
start;
|
|
113855
|
+
end;
|
|
113856
|
+
contentType;
|
|
113857
|
+
full;
|
|
113858
|
+
fetch;
|
|
113859
|
+
accessToken;
|
|
113860
|
+
constructor(url, start, end, contentType, full, customFetch, accessToken) {
|
|
113861
|
+
super([]);
|
|
113862
|
+
this.url = url;
|
|
113863
|
+
this.start = start;
|
|
113864
|
+
this.end = end;
|
|
113865
|
+
this.contentType = contentType;
|
|
113866
|
+
this.full = full;
|
|
113867
|
+
this.fetch = customFetch;
|
|
113868
|
+
this.accessToken = accessToken;
|
|
113869
|
+
}
|
|
113870
|
+
get size() {
|
|
113871
|
+
return this.end - this.start;
|
|
113872
|
+
}
|
|
113873
|
+
get type() {
|
|
113874
|
+
return this.contentType;
|
|
113875
|
+
}
|
|
113876
|
+
slice(start = 0, end = this.size) {
|
|
113877
|
+
const slice = new WebBlob(
|
|
113878
|
+
this.url,
|
|
113879
|
+
this.start + start,
|
|
113880
|
+
Math.min(this.start + end, this.end),
|
|
113881
|
+
this.contentType,
|
|
113882
|
+
start === 0 && end === this.size ? this.full : false,
|
|
113883
|
+
this.fetch,
|
|
113884
|
+
this.accessToken
|
|
113885
|
+
);
|
|
113886
|
+
return slice;
|
|
113887
|
+
}
|
|
113888
|
+
async arrayBuffer() {
|
|
113889
|
+
const result = await this.fetchRange();
|
|
113890
|
+
return result.arrayBuffer();
|
|
113891
|
+
}
|
|
113892
|
+
async text() {
|
|
113893
|
+
const result = await this.fetchRange();
|
|
113894
|
+
return result.text();
|
|
113895
|
+
}
|
|
113896
|
+
stream() {
|
|
113897
|
+
const stream = new TransformStream();
|
|
113898
|
+
this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
113899
|
+
return stream.readable;
|
|
113900
|
+
}
|
|
113901
|
+
fetchRange() {
|
|
113902
|
+
const fetch2 = this.fetch;
|
|
113903
|
+
if (this.full) {
|
|
113904
|
+
return fetch2(this.url, {
|
|
113905
|
+
...this.accessToken && {
|
|
113906
|
+
headers: {
|
|
113907
|
+
Authorization: `Bearer ${this.accessToken}`
|
|
113908
|
+
}
|
|
113909
|
+
}
|
|
113910
|
+
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
113911
|
+
}
|
|
113912
|
+
return fetch2(this.url, {
|
|
113913
|
+
headers: {
|
|
113914
|
+
Range: `bytes=${this.start}-${this.end - 1}`,
|
|
113915
|
+
...this.accessToken && { Authorization: `Bearer ${this.accessToken}` }
|
|
113916
|
+
}
|
|
113917
|
+
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
113918
|
+
}
|
|
113919
|
+
};
|
|
113920
|
+
|
|
113921
|
+
// src/utils/parseLinkHeader.ts
|
|
113922
|
+
function parseLinkHeader(header) {
|
|
113923
|
+
const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g;
|
|
113924
|
+
return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url]));
|
|
113925
|
+
}
|
|
113926
|
+
|
|
113927
|
+
// src/lib/file-download-info.ts
|
|
113928
|
+
async function fileDownloadInfo(params) {
|
|
113929
|
+
const accessToken = checkCredentials(params);
|
|
113930
|
+
const repoId = toRepoId(params.repo);
|
|
113931
|
+
const hubUrl = params.hubUrl ?? HUB_URL;
|
|
113932
|
+
const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}/${encodeURIComponent(params.revision ?? "main")}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : "");
|
|
113933
|
+
const resp = await (params.fetch ?? fetch)(url, {
|
|
113934
|
+
method: "GET",
|
|
113935
|
+
headers: {
|
|
113936
|
+
...accessToken && {
|
|
113937
|
+
Authorization: `Bearer ${accessToken}`
|
|
113938
|
+
},
|
|
113939
|
+
Range: "bytes=0-0",
|
|
113940
|
+
Accept: "application/vnd.xet-fileinfo+json, */*"
|
|
113941
|
+
}
|
|
113942
|
+
});
|
|
113943
|
+
if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") {
|
|
113944
|
+
return null;
|
|
113945
|
+
}
|
|
113946
|
+
if (!resp.ok) {
|
|
113947
|
+
throw await createApiError(resp);
|
|
113948
|
+
}
|
|
113949
|
+
let size;
|
|
113950
|
+
let xetInfo;
|
|
113951
|
+
if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) {
|
|
113952
|
+
size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid");
|
|
113953
|
+
if (isNaN(size)) {
|
|
113954
|
+
throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header");
|
|
113955
|
+
}
|
|
113956
|
+
const hash2 = resp.headers.get("X-Xet-Hash");
|
|
113957
|
+
const links = parseLinkHeader(resp.headers.get("Link") ?? "");
|
|
113958
|
+
const reconstructionUrl = (() => {
|
|
113959
|
+
try {
|
|
113960
|
+
return new URL(links["xet-reconstruction-info"]);
|
|
113961
|
+
} catch {
|
|
113962
|
+
return null;
|
|
113963
|
+
}
|
|
113964
|
+
})();
|
|
113965
|
+
const refreshUrl = (() => {
|
|
113966
|
+
try {
|
|
113967
|
+
return new URL(links["xet-auth"]);
|
|
113968
|
+
} catch {
|
|
113969
|
+
return null;
|
|
113970
|
+
}
|
|
113971
|
+
})();
|
|
113972
|
+
if (!hash2) {
|
|
113973
|
+
throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header");
|
|
113974
|
+
}
|
|
113975
|
+
if (!reconstructionUrl || !refreshUrl) {
|
|
113976
|
+
throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header");
|
|
113977
|
+
}
|
|
113978
|
+
xetInfo = {
|
|
113979
|
+
hash: hash2,
|
|
113980
|
+
refreshUrl,
|
|
113981
|
+
reconstructionUrl
|
|
113982
|
+
};
|
|
113983
|
+
}
|
|
113984
|
+
if (size === void 0 || isNaN(size)) {
|
|
113985
|
+
const contentRangeHeader = resp.headers.get("content-range");
|
|
113986
|
+
if (!contentRangeHeader) {
|
|
113987
|
+
throw new InvalidApiResponseFormatError("Expected size information");
|
|
113988
|
+
}
|
|
113989
|
+
const [, parsedSize] = contentRangeHeader.split("/");
|
|
113990
|
+
size = parseInt(parsedSize);
|
|
113991
|
+
if (isNaN(size)) {
|
|
113992
|
+
throw new InvalidApiResponseFormatError("Invalid file size received");
|
|
113993
|
+
}
|
|
113994
|
+
}
|
|
113995
|
+
const etag = resp.headers.get("X-Linked-ETag") ?? resp.headers.get("ETag") ?? void 0;
|
|
113996
|
+
if (!etag) {
|
|
113997
|
+
throw new InvalidApiResponseFormatError("Expected ETag");
|
|
113998
|
+
}
|
|
113999
|
+
return {
|
|
114000
|
+
etag,
|
|
114001
|
+
size,
|
|
114002
|
+
xet: xetInfo,
|
|
114003
|
+
// Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it.
|
|
114004
|
+
url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url
|
|
114005
|
+
};
|
|
114006
|
+
}
|
|
114007
|
+
|
|
114008
|
+
// src/lib/download-file.ts
|
|
114009
|
+
async function downloadFile(params) {
|
|
114010
|
+
const accessToken = checkCredentials(params);
|
|
114011
|
+
const info = params.downloadInfo ?? await fileDownloadInfo({
|
|
114012
|
+
accessToken,
|
|
114013
|
+
repo: params.repo,
|
|
114014
|
+
path: params.path,
|
|
114015
|
+
revision: params.revision,
|
|
114016
|
+
hubUrl: params.hubUrl,
|
|
114017
|
+
fetch: params.fetch,
|
|
114018
|
+
raw: params.raw
|
|
114019
|
+
});
|
|
114020
|
+
if (!info) {
|
|
114021
|
+
return null;
|
|
114022
|
+
}
|
|
114023
|
+
if (info.xet && params.xet) {
|
|
114024
|
+
return new XetBlob({
|
|
114025
|
+
refreshUrl: info.xet.refreshUrl.href,
|
|
114026
|
+
reconstructionUrl: info.xet.reconstructionUrl.href,
|
|
114027
|
+
fetch: params.fetch,
|
|
114028
|
+
accessToken,
|
|
114029
|
+
size: info.size
|
|
114030
|
+
});
|
|
114031
|
+
}
|
|
114032
|
+
return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken);
|
|
114033
|
+
}
|
|
114034
|
+
|
|
114035
|
+
// src/lib/list-files.ts
|
|
114036
|
+
async function* listFiles(params) {
|
|
114037
|
+
const accessToken = checkCredentials(params);
|
|
114038
|
+
const repoId = toRepoId(params.repo);
|
|
114039
|
+
let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree/${params.revision || "main"}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`;
|
|
114040
|
+
while (url) {
|
|
114041
|
+
const res = await (params.fetch ?? fetch)(url, {
|
|
114042
|
+
headers: {
|
|
114043
|
+
accept: "application/json",
|
|
114044
|
+
...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0
|
|
114045
|
+
}
|
|
114046
|
+
});
|
|
114047
|
+
if (!res.ok) {
|
|
114048
|
+
throw await createApiError(res);
|
|
114049
|
+
}
|
|
114050
|
+
const items = await res.json();
|
|
114051
|
+
for (const item of items) {
|
|
114052
|
+
yield item;
|
|
114053
|
+
}
|
|
114054
|
+
const linkHeader = res.headers.get("Link");
|
|
114055
|
+
url = linkHeader ? parseLinkHeader(linkHeader).next : void 0;
|
|
114056
|
+
}
|
|
114057
|
+
}
|
|
114058
|
+
|
|
114059
|
+
const DIR_BASED_ENGINES = new Set(["exllamav3", "mlx-lm"]);
|
|
114060
|
+
const FLAG_BASED_ENGINE_ARGS = {
|
|
114061
|
+
"llama.cpp": "--chat-template-file",
|
|
114062
|
+
sglang: "--chat-template",
|
|
114063
|
+
vllm: "--chat-template"
|
|
114064
|
+
};
|
|
114065
|
+
function getChatTemplateLocalPath(targetDirectory) {
|
|
114066
|
+
return path$1.join(targetDirectory, CHAT_TEMPLATE_LOCAL_FILE_NAME);
|
|
114067
|
+
}
|
|
114068
|
+
async function fetchTemplateContent({ huggingFaceToken, override }) {
|
|
114069
|
+
const accessToken = huggingFaceToken ?? process.env.HF_TOKEN ?? undefined;
|
|
114070
|
+
const filePath = override.filePath || CHAT_TEMPLATE_DEFAULT_FILE_PATH;
|
|
114071
|
+
const blob = await downloadFile({
|
|
114072
|
+
accessToken,
|
|
114073
|
+
path: filePath,
|
|
114074
|
+
repo: override.repo
|
|
114075
|
+
});
|
|
114076
|
+
if (!blob) {
|
|
114077
|
+
throw new Error(`Chat template file not found: ${override.repo}/${filePath}`);
|
|
114078
|
+
}
|
|
114079
|
+
const content = await blob.text();
|
|
114080
|
+
if (!content.trim()) {
|
|
114081
|
+
throw new Error(`Chat template file is empty: ${override.repo}/${filePath}`);
|
|
114082
|
+
}
|
|
114083
|
+
return content;
|
|
114084
|
+
}
|
|
114085
|
+
/**
|
|
114086
|
+
* Materializes the model's chat template override into the model directory:
|
|
114087
|
+
* - Always writes the canonical copy for flag-based engines.
|
|
114088
|
+
* - For engines that read the model directory (exllamav3, mlx-lm), also writes
|
|
114089
|
+
* the transformers-style `chat_template.jinja` drop-in.
|
|
114090
|
+
* - When no override is configured, removes any previously materialized files,
|
|
114091
|
+
* restoring the model repo's own `chat_template.jinja` if we replaced it.
|
|
114092
|
+
*/
|
|
114093
|
+
async function materializeChatTemplate({ engine, huggingFaceToken, model, targetDirectory }) {
|
|
114094
|
+
const override = model.chatTemplate ?? null;
|
|
114095
|
+
const canonicalPath = getChatTemplateLocalPath(targetDirectory);
|
|
114096
|
+
const dropInPath = path$1.join(targetDirectory, CHAT_TEMPLATE_DEFAULT_FILE_PATH);
|
|
114097
|
+
if (!override) {
|
|
114098
|
+
const markerPath = `${canonicalPath}.override`;
|
|
114099
|
+
if (require$$3$4.existsSync(markerPath)) {
|
|
114100
|
+
await require$$0$m.rm(dropInPath, { force: true });
|
|
114101
|
+
await require$$0$m.rm(markerPath, { force: true });
|
|
114102
|
+
if (model.source.type === "huggingface") {
|
|
114103
|
+
await restoreOriginalTemplate({
|
|
114104
|
+
huggingFaceToken,
|
|
114105
|
+
modelSlug: model.source.slug,
|
|
114106
|
+
targetDirectory
|
|
114107
|
+
});
|
|
114108
|
+
}
|
|
114109
|
+
}
|
|
114110
|
+
await require$$0$m.rm(canonicalPath, { force: true });
|
|
114111
|
+
return;
|
|
114112
|
+
}
|
|
114113
|
+
const content = override.type === "inline"
|
|
114114
|
+
? override.content
|
|
114115
|
+
: await fetchTemplateContent({ huggingFaceToken, override });
|
|
114116
|
+
await require$$0$m.mkdir(targetDirectory, { recursive: true });
|
|
114117
|
+
await require$$0$m.writeFile(canonicalPath, content, "utf8");
|
|
114118
|
+
if (DIR_BASED_ENGINES.has(engine)) {
|
|
114119
|
+
await require$$0$m.writeFile(dropInPath, content, "utf8");
|
|
114120
|
+
await require$$0$m.writeFile(`${canonicalPath}.override`, "1", "utf8");
|
|
114121
|
+
}
|
|
114122
|
+
}
|
|
114123
|
+
async function restoreOriginalTemplate({ huggingFaceToken, modelSlug, targetDirectory }) {
|
|
114124
|
+
const accessToken = huggingFaceToken ?? process.env.HF_TOKEN ?? undefined;
|
|
114125
|
+
try {
|
|
114126
|
+
const blob = await downloadFile({
|
|
114127
|
+
accessToken,
|
|
114128
|
+
path: CHAT_TEMPLATE_DEFAULT_FILE_PATH,
|
|
114129
|
+
repo: modelSlug.split(":")[0].split("@")[0]
|
|
114130
|
+
});
|
|
114131
|
+
if (blob) {
|
|
114132
|
+
const content = await blob.text();
|
|
114133
|
+
if (content.trim()) {
|
|
114134
|
+
await require$$0$m.writeFile(path$1.join(targetDirectory, CHAT_TEMPLATE_DEFAULT_FILE_PATH), content, "utf8");
|
|
114135
|
+
}
|
|
114136
|
+
}
|
|
114137
|
+
}
|
|
114138
|
+
catch (error) {
|
|
114139
|
+
console.warn("[chatTemplate] Failed to restore original chat_template.jinja after override removal:", asError(error).message);
|
|
114140
|
+
}
|
|
114141
|
+
}
|
|
114142
|
+
/**
|
|
114143
|
+
* CLI arguments applying the materialized chat template for flag-based engines.
|
|
114144
|
+
* Returns an empty array when no override exists, the engine applies templates
|
|
114145
|
+
* from the model directory, or the engine does not support overrides.
|
|
114146
|
+
*/
|
|
114147
|
+
async function getChatTemplateEngineArgs({ engine, model, targetDirectory }) {
|
|
114148
|
+
if (!model.chatTemplate)
|
|
114149
|
+
return [];
|
|
114150
|
+
const flag = FLAG_BASED_ENGINE_ARGS[engine];
|
|
114151
|
+
if (!flag) {
|
|
114152
|
+
if (engine === "tensorrt-llm") {
|
|
114153
|
+
console.warn("[chatTemplate] TensorRT-LLM does not support chat template overrides; ignoring");
|
|
114154
|
+
}
|
|
114155
|
+
return [];
|
|
114156
|
+
}
|
|
114157
|
+
const templatePath = getChatTemplateLocalPath(targetDirectory);
|
|
114158
|
+
if (!require$$3$4.existsSync(templatePath)) {
|
|
114159
|
+
console.warn(`[chatTemplate] Template file missing for ${engine}; continuing with embedded template`);
|
|
114160
|
+
return [];
|
|
114161
|
+
}
|
|
114162
|
+
return [flag, templatePath];
|
|
114163
|
+
}
|
|
114164
|
+
/**
|
|
114165
|
+
* Post-start verification for llama.cpp: compares the template the server
|
|
114166
|
+
* reports via `/props` against the materialized override file. Returns a
|
|
114167
|
+
* warning message on mismatch, or null when the template applied cleanly
|
|
114168
|
+
* (or verification is not applicable).
|
|
114169
|
+
*/
|
|
114170
|
+
async function verifyEngineChatTemplate({ engine, enginePort, logger, model, targetDirectory }) {
|
|
114171
|
+
if (!model.chatTemplate || engine !== "llama.cpp")
|
|
114172
|
+
return null;
|
|
114173
|
+
try {
|
|
114174
|
+
const templatePath = getChatTemplateLocalPath(targetDirectory);
|
|
114175
|
+
if (!require$$3$4.existsSync(templatePath)) {
|
|
114176
|
+
return "Chat template override file missing on disk";
|
|
114177
|
+
}
|
|
114178
|
+
const [propsResponse, expected] = await Promise.all([
|
|
114179
|
+
fetch(`http://localhost:${enginePort}/props`, {
|
|
114180
|
+
signal: AbortSignal.timeout(5000)
|
|
114181
|
+
}),
|
|
114182
|
+
require$$0$m.readFile(templatePath, "utf8")
|
|
114183
|
+
]);
|
|
114184
|
+
if (!propsResponse.ok) {
|
|
114185
|
+
return `Chat template verification unavailable: /props returned ${propsResponse.status}`;
|
|
114186
|
+
}
|
|
114187
|
+
const props = (await propsResponse.json());
|
|
114188
|
+
const served = typeof props.chat_template === "string" ? props.chat_template.trim() : "";
|
|
114189
|
+
if (served !== expected.trim()) {
|
|
114190
|
+
return "Chat template override did not apply: served template differs from configured template";
|
|
114191
|
+
}
|
|
114192
|
+
return null;
|
|
114193
|
+
}
|
|
114194
|
+
catch (error) {
|
|
114195
|
+
logger.warn("Chat template verification failed", {
|
|
114196
|
+
error: asError(error)
|
|
114197
|
+
});
|
|
114198
|
+
return null;
|
|
114199
|
+
}
|
|
114200
|
+
}
|
|
114201
|
+
|
|
114202
|
+
const SECRET_FLAGS = new Set([
|
|
114203
|
+
"api-key",
|
|
114204
|
+
"auth-token",
|
|
114205
|
+
"hf-token",
|
|
114206
|
+
"key",
|
|
114207
|
+
"password",
|
|
114208
|
+
"secret",
|
|
114209
|
+
"token"
|
|
114210
|
+
]);
|
|
114211
|
+
function redactSecretArgs(args) {
|
|
114212
|
+
return args.map((arg, index) => {
|
|
114213
|
+
const equalsIndex = arg.indexOf("=");
|
|
114214
|
+
if (arg.startsWith("--") && equalsIndex > 0) {
|
|
114215
|
+
const flag = arg.slice(2, equalsIndex);
|
|
114216
|
+
if (SECRET_FLAGS.has(flag)) {
|
|
114217
|
+
return `${arg.slice(0, equalsIndex + 1)}***`;
|
|
114218
|
+
}
|
|
114219
|
+
}
|
|
114220
|
+
const previous = args[index - 1];
|
|
114221
|
+
if (previous &&
|
|
114222
|
+
previous.startsWith("--") &&
|
|
114223
|
+
!previous.includes("=") &&
|
|
114224
|
+
SECRET_FLAGS.has(previous.slice(2))) {
|
|
114225
|
+
return "***";
|
|
114226
|
+
}
|
|
114227
|
+
return arg;
|
|
114228
|
+
});
|
|
114229
|
+
}
|
|
114230
|
+
async function createEngineProcess({ args, bin, logger }) {
|
|
114231
|
+
logger.info("Starting engine process", {
|
|
114232
|
+
command: { args: redactSecretArgs(args), bin }
|
|
114233
|
+
});
|
|
114234
|
+
const processManager = new ProcessManager({ args, command: bin });
|
|
114235
|
+
await processManager.start();
|
|
114236
|
+
return processManager;
|
|
114237
|
+
}
|
|
114238
|
+
|
|
114239
|
+
function parseExtraArgs(extraArgs) {
|
|
114240
|
+
if (!Array.isArray(extraArgs) ||
|
|
114241
|
+
!extraArgs.every((value) => typeof value === "string")) {
|
|
114242
|
+
return [];
|
|
114243
|
+
}
|
|
114244
|
+
return extraArgs.flatMap(tokenizeShellLine);
|
|
114245
|
+
}
|
|
114246
|
+
function tokenizeShellLine(input) {
|
|
114247
|
+
const tokens = [];
|
|
114248
|
+
let buffer = "";
|
|
114249
|
+
let inQuote = null;
|
|
114250
|
+
let hasBuffer = false;
|
|
114251
|
+
for (let index = 0; index < input.length; index += 1) {
|
|
114252
|
+
const char = input[index];
|
|
114253
|
+
if (inQuote) {
|
|
114254
|
+
if (char === inQuote) {
|
|
114255
|
+
inQuote = null;
|
|
114256
|
+
}
|
|
114257
|
+
else {
|
|
114258
|
+
buffer += char;
|
|
114259
|
+
}
|
|
114260
|
+
hasBuffer = true;
|
|
114261
|
+
}
|
|
114262
|
+
else if (char === '"' || char === "'") {
|
|
114263
|
+
inQuote = char;
|
|
114264
|
+
hasBuffer = true;
|
|
114265
|
+
}
|
|
114266
|
+
else if (char === " " || char === "\t") {
|
|
114267
|
+
if (hasBuffer) {
|
|
114268
|
+
tokens.push(buffer);
|
|
114269
|
+
buffer = "";
|
|
114270
|
+
hasBuffer = false;
|
|
114271
|
+
}
|
|
114272
|
+
}
|
|
114273
|
+
else {
|
|
114274
|
+
buffer += char;
|
|
114275
|
+
hasBuffer = true;
|
|
114276
|
+
}
|
|
114277
|
+
}
|
|
114278
|
+
if (hasBuffer) {
|
|
114279
|
+
tokens.push(buffer);
|
|
114280
|
+
}
|
|
114281
|
+
return tokens;
|
|
114282
|
+
}
|
|
114283
|
+
|
|
114284
|
+
const balanced = (a, b, str) => {
|
|
114285
|
+
const ma = a instanceof RegExp ? maybeMatch(a, str) : a;
|
|
114286
|
+
const mb = b instanceof RegExp ? maybeMatch(b, str) : b;
|
|
114287
|
+
const r = ma !== null && mb != null && range(ma, mb, str);
|
|
114288
|
+
return (r && {
|
|
114289
|
+
start: r[0],
|
|
114290
|
+
end: r[1],
|
|
114291
|
+
pre: str.slice(0, r[0]),
|
|
114292
|
+
body: str.slice(r[0] + ma.length, r[1]),
|
|
114293
|
+
post: str.slice(r[1] + mb.length),
|
|
114294
|
+
});
|
|
114295
|
+
};
|
|
114296
|
+
const maybeMatch = (reg, str) => {
|
|
114297
|
+
const m = str.match(reg);
|
|
114298
|
+
return m ? m[0] : null;
|
|
114299
|
+
};
|
|
114300
|
+
const range = (a, b, str) => {
|
|
114301
|
+
let begs, beg, left, right = undefined, result;
|
|
114302
|
+
let ai = str.indexOf(a);
|
|
114303
|
+
let bi = str.indexOf(b, ai + 1);
|
|
114304
|
+
let i = ai;
|
|
114305
|
+
if (ai >= 0 && bi > 0) {
|
|
114306
|
+
if (a === b) {
|
|
114307
|
+
return [ai, bi];
|
|
114308
|
+
}
|
|
114309
|
+
begs = [];
|
|
114310
|
+
left = str.length;
|
|
114311
|
+
while (i >= 0 && !result) {
|
|
114312
|
+
if (i === ai) {
|
|
114313
|
+
begs.push(i);
|
|
114314
|
+
ai = str.indexOf(a, i + 1);
|
|
114315
|
+
}
|
|
114316
|
+
else if (begs.length === 1) {
|
|
114317
|
+
const r = begs.pop();
|
|
114318
|
+
if (r !== undefined)
|
|
114319
|
+
result = [r, bi];
|
|
114320
|
+
}
|
|
114321
|
+
else {
|
|
114322
|
+
beg = begs.pop();
|
|
114323
|
+
if (beg !== undefined && beg < left) {
|
|
114324
|
+
left = beg;
|
|
114325
|
+
right = bi;
|
|
114326
|
+
}
|
|
114327
|
+
bi = str.indexOf(b, i + 1);
|
|
114328
|
+
}
|
|
114329
|
+
i = ai < bi && ai >= 0 ? ai : bi;
|
|
114330
|
+
}
|
|
114331
|
+
if (begs.length && right !== undefined) {
|
|
114332
|
+
result = [left, right];
|
|
114333
|
+
}
|
|
114334
|
+
}
|
|
114335
|
+
return result;
|
|
114336
|
+
};
|
|
114337
|
+
|
|
114338
|
+
const escSlash = '\0SLASH' + Math.random() + '\0';
|
|
114339
|
+
const escOpen = '\0OPEN' + Math.random() + '\0';
|
|
114340
|
+
const escClose = '\0CLOSE' + Math.random() + '\0';
|
|
114341
|
+
const escComma = '\0COMMA' + Math.random() + '\0';
|
|
114342
|
+
const escPeriod = '\0PERIOD' + Math.random() + '\0';
|
|
114343
|
+
const escSlashPattern = new RegExp(escSlash, 'g');
|
|
114344
|
+
const escOpenPattern = new RegExp(escOpen, 'g');
|
|
114345
|
+
const escClosePattern = new RegExp(escClose, 'g');
|
|
114346
|
+
const escCommaPattern = new RegExp(escComma, 'g');
|
|
114347
|
+
const escPeriodPattern = new RegExp(escPeriod, 'g');
|
|
114348
|
+
const slashPattern = /\\\\/g;
|
|
114349
|
+
const openPattern = /\\{/g;
|
|
114350
|
+
const closePattern = /\\}/g;
|
|
114351
|
+
const commaPattern = /\\,/g;
|
|
114352
|
+
const periodPattern = /\\\./g;
|
|
114353
|
+
const EXPANSION_MAX = 100_000;
|
|
114354
|
+
// `EXPANSION_MAX` caps the *number* of expansions, but not their length. An
|
|
114355
|
+
// input like `'{a,b}'.repeat(1500)` stays under that count - its output is
|
|
114356
|
+
// truncated to 100k results - while making every result ~1500 characters
|
|
114357
|
+
// long. The result set, and the intermediate arrays built while combining
|
|
114358
|
+
// brace sets, then grow large enough to exhaust memory and crash the process
|
|
114359
|
+
// (CVE-2026-14257). `EXPANSION_MAX_LENGTH` bounds the total number of
|
|
114360
|
+
// characters the accumulator may hold at any point, so memory stays flat no
|
|
114361
|
+
// matter how many brace groups are chained. The limit sits well above any
|
|
114362
|
+
// realistic expansion (100k results hitting `EXPANSION_MAX` measure ~1M
|
|
114363
|
+
// characters) so legitimate input is unaffected.
|
|
114364
|
+
const EXPANSION_MAX_LENGTH = 4_000_000;
|
|
114365
|
+
function numeric(str) {
|
|
114366
|
+
return !isNaN(str) ? parseInt(str, 10) : str.charCodeAt(0);
|
|
114367
|
+
}
|
|
114368
|
+
function escapeBraces(str) {
|
|
114369
|
+
return str
|
|
114370
|
+
.replace(slashPattern, escSlash)
|
|
114371
|
+
.replace(openPattern, escOpen)
|
|
114372
|
+
.replace(closePattern, escClose)
|
|
114373
|
+
.replace(commaPattern, escComma)
|
|
114374
|
+
.replace(periodPattern, escPeriod);
|
|
114375
|
+
}
|
|
114376
|
+
function unescapeBraces(str) {
|
|
114377
|
+
return str
|
|
114378
|
+
.replace(escSlashPattern, '\\')
|
|
114379
|
+
.replace(escOpenPattern, '{')
|
|
114380
|
+
.replace(escClosePattern, '}')
|
|
114381
|
+
.replace(escCommaPattern, ',')
|
|
114382
|
+
.replace(escPeriodPattern, '.');
|
|
114383
|
+
}
|
|
114384
|
+
/**
|
|
114385
|
+
* Basically just str.split(","), but handling cases
|
|
114386
|
+
* where we have nested braced sections, which should be
|
|
114387
|
+
* treated as individual members, like {a,{b,c},d}
|
|
114388
|
+
*/
|
|
114389
|
+
function parseCommaParts(str) {
|
|
114390
|
+
if (!str) {
|
|
114391
|
+
return [''];
|
|
114392
|
+
}
|
|
114393
|
+
const parts = [];
|
|
114394
|
+
const m = balanced('{', '}', str);
|
|
114395
|
+
if (!m) {
|
|
114396
|
+
return str.split(',');
|
|
114397
|
+
}
|
|
114398
|
+
const { pre, body, post } = m;
|
|
114399
|
+
const p = pre.split(',');
|
|
114400
|
+
p[p.length - 1] += '{' + body + '}';
|
|
114401
|
+
const postParts = parseCommaParts(post);
|
|
114402
|
+
if (post.length) {
|
|
114403
|
+
p[p.length - 1] += postParts.shift();
|
|
114404
|
+
p.push.apply(p, postParts);
|
|
114405
|
+
}
|
|
114406
|
+
parts.push.apply(parts, p);
|
|
114407
|
+
return parts;
|
|
114408
|
+
}
|
|
114409
|
+
function expand(str, options = {}) {
|
|
114410
|
+
if (!str) {
|
|
114411
|
+
return [];
|
|
114412
|
+
}
|
|
114413
|
+
const { max = EXPANSION_MAX, maxLength = EXPANSION_MAX_LENGTH } = options;
|
|
114414
|
+
// I don't know why Bash 4.3 does this, but it does.
|
|
114415
|
+
// Anything starting with {} will have the first two bytes preserved
|
|
114416
|
+
// but *only* at the top level, so {},a}b will not expand to anything,
|
|
114417
|
+
// but a{},b}c will be expanded to [a}c,abc].
|
|
114418
|
+
// One could argue that this is a bug in Bash, but since the goal of
|
|
114419
|
+
// this module is to match Bash's rules, we escape a leading {}
|
|
114420
|
+
if (str.slice(0, 2) === '{}') {
|
|
114421
|
+
str = '\\{\\}' + str.slice(2);
|
|
114422
|
+
}
|
|
114423
|
+
return expand_(escapeBraces(str), max, maxLength, true).map(unescapeBraces);
|
|
114424
|
+
}
|
|
114425
|
+
function embrace(str) {
|
|
114426
|
+
return '{' + str + '}';
|
|
114427
|
+
}
|
|
114428
|
+
function isPadded(el) {
|
|
114429
|
+
return /^-?0\d/.test(el);
|
|
114430
|
+
}
|
|
114431
|
+
function lte(i, y) {
|
|
114432
|
+
return i <= y;
|
|
114433
|
+
}
|
|
114434
|
+
function gte(i, y) {
|
|
114435
|
+
return i >= y;
|
|
114436
|
+
}
|
|
114437
|
+
// Build `{ acc[a] + pre + values[v] }` for every combination, capping the
|
|
114438
|
+
// number of results at `max` and the total number of characters at `maxLength`.
|
|
114439
|
+
// This is the one place output grows, so bounding it here keeps the single
|
|
114440
|
+
// accumulator - and therefore memory - flat regardless of how many brace groups
|
|
114441
|
+
// are combined (CVE-2026-14257).
|
|
114442
|
+
function combine(acc, pre, values, max, maxLength, dropEmpties) {
|
|
114443
|
+
const out = [];
|
|
114444
|
+
let length = 0;
|
|
114445
|
+
for (let a = 0; a < acc.length; a++) {
|
|
114446
|
+
for (let v = 0; v < values.length; v++) {
|
|
114447
|
+
if (out.length >= max)
|
|
114448
|
+
return out;
|
|
114449
|
+
const expansion = acc[a] + pre + values[v];
|
|
114450
|
+
// Bash drops empty results at the top level. Skip them before they count
|
|
114451
|
+
// against `max`, so `max` bounds the number of *kept* results.
|
|
114452
|
+
if (dropEmpties && !expansion)
|
|
114453
|
+
continue;
|
|
114454
|
+
if (length + expansion.length > maxLength)
|
|
114455
|
+
return out;
|
|
114456
|
+
out.push(expansion);
|
|
114457
|
+
length += expansion.length;
|
|
114458
|
+
}
|
|
114459
|
+
}
|
|
114460
|
+
return out;
|
|
114461
|
+
}
|
|
114462
|
+
// The expansion values of a single numeric (`1..5`) or alphabetic (`a..e..2`)
|
|
114463
|
+
// sequence body.
|
|
114464
|
+
function expandSequence(body, isAlphaSequence, max, maxLength) {
|
|
114465
|
+
const n = body.split(/\.\./);
|
|
114466
|
+
const N = [];
|
|
114467
|
+
// A sequence body always splits into two or three parts, but the compiler
|
|
114468
|
+
// can't know that.
|
|
114469
|
+
/* c8 ignore start */
|
|
114470
|
+
if (n[0] === undefined || n[1] === undefined) {
|
|
114471
|
+
return N;
|
|
114472
|
+
}
|
|
114473
|
+
/* c8 ignore stop */
|
|
114474
|
+
const x = numeric(n[0]);
|
|
114475
|
+
const y = numeric(n[1]);
|
|
114476
|
+
const width = Math.max(n[0].length, n[1].length);
|
|
114477
|
+
let incr = n.length === 3 && n[2] !== undefined ?
|
|
114478
|
+
Math.max(Math.abs(numeric(n[2])), 1)
|
|
114479
|
+
: 1;
|
|
114480
|
+
let test = lte;
|
|
114481
|
+
const reverse = y < x;
|
|
114482
|
+
if (reverse) {
|
|
114483
|
+
incr *= -1;
|
|
114484
|
+
test = gte;
|
|
114485
|
+
}
|
|
114486
|
+
const pad = n.some(isPadded);
|
|
114487
|
+
let length = 0;
|
|
114488
|
+
for (let i = x; test(i, y) && N.length < max; i += incr) {
|
|
114489
|
+
let c;
|
|
114490
|
+
if (isAlphaSequence) {
|
|
114491
|
+
c = String.fromCharCode(i);
|
|
114492
|
+
if (c === '\\') {
|
|
114493
|
+
c = '';
|
|
114494
|
+
}
|
|
114495
|
+
}
|
|
114496
|
+
else {
|
|
114497
|
+
c = String(i);
|
|
114498
|
+
if (pad) {
|
|
114499
|
+
const need = width - c.length;
|
|
114500
|
+
if (need > 0) {
|
|
114501
|
+
const z = new Array(need + 1).join('0');
|
|
114502
|
+
if (i < 0) {
|
|
114503
|
+
c = '-' + z + c.slice(1);
|
|
114504
|
+
}
|
|
114505
|
+
else {
|
|
114506
|
+
c = z + c;
|
|
114507
|
+
}
|
|
114508
|
+
}
|
|
114509
|
+
}
|
|
114510
|
+
}
|
|
114511
|
+
if (length + c.length > maxLength)
|
|
114512
|
+
break;
|
|
114513
|
+
N.push(c);
|
|
114514
|
+
length += c.length;
|
|
114515
|
+
}
|
|
114516
|
+
return N;
|
|
114517
|
+
}
|
|
114518
|
+
function expand_(str, max, maxLength, isTop) {
|
|
114519
|
+
// Consume the string's top-level brace groups left to right, threading a
|
|
114520
|
+
// running set of combined prefixes (`acc`). Expanding the tail iteratively -
|
|
114521
|
+
// rather than recursing on `m.post` once per group - keeps the native stack
|
|
114522
|
+
// depth constant, so deeply chained input (`'{a,b}'.repeat(3000)`) can no
|
|
114523
|
+
// longer overflow the stack, and leaves a single accumulator whose size
|
|
114524
|
+
// `maxLength` bounds directly (CVE-2026-14257).
|
|
114525
|
+
let acc = [''];
|
|
114526
|
+
// Bash drops empty results, but only when the *first* top-level group is a
|
|
114527
|
+
// comma set - a sequence like `{a..\}` may legitimately yield ''. The drop
|
|
114528
|
+
// is on the final strings, so it is applied to whichever `combine` produces
|
|
114529
|
+
// them (the one with no brace set left in the tail).
|
|
114530
|
+
let dropEmpties = false;
|
|
114531
|
+
let firstGroup = true;
|
|
114532
|
+
for (;;) {
|
|
114533
|
+
const m = balanced('{', '}', str);
|
|
114534
|
+
// No brace set left: the rest of the string is literal.
|
|
114535
|
+
if (!m) {
|
|
114536
|
+
return combine(acc, str, [''], max, maxLength, dropEmpties);
|
|
114537
|
+
}
|
|
114538
|
+
// no need to expand pre, since it is guaranteed to be free of brace-sets
|
|
114539
|
+
const pre = m.pre;
|
|
114540
|
+
if (/\$$/.test(pre)) {
|
|
114541
|
+
acc = combine(acc, pre + '{' + m.body + '}', [''], max, maxLength, dropEmpties && !m.post.length);
|
|
114542
|
+
firstGroup = false;
|
|
114543
|
+
if (!m.post.length)
|
|
114544
|
+
break;
|
|
114545
|
+
str = m.post;
|
|
114546
|
+
continue;
|
|
114547
|
+
}
|
|
114548
|
+
const isNumericSequence = /^-?\d+\.\.-?\d+(?:\.\.-?\d+)?$/.test(m.body);
|
|
114549
|
+
const isAlphaSequence = /^[a-zA-Z]\.\.[a-zA-Z](?:\.\.-?\d+)?$/.test(m.body);
|
|
114550
|
+
const isSequence = isNumericSequence || isAlphaSequence;
|
|
114551
|
+
const isOptions = m.body.indexOf(',') >= 0;
|
|
114552
|
+
if (!isSequence && !isOptions) {
|
|
114553
|
+
// {a},b}
|
|
114554
|
+
if (m.post.match(/,(?!,).*\}/)) {
|
|
114555
|
+
str = m.pre + '{' + m.body + escClose + m.post;
|
|
114556
|
+
isTop = true;
|
|
114557
|
+
continue;
|
|
114558
|
+
}
|
|
114559
|
+
// Nothing here expands, so the whole remaining string is literal.
|
|
114560
|
+
return combine(acc, pre + '{' + m.body + '}' + m.post, [''], max, maxLength, dropEmpties);
|
|
114561
|
+
}
|
|
114562
|
+
if (firstGroup) {
|
|
114563
|
+
dropEmpties = isTop && !isSequence;
|
|
114564
|
+
firstGroup = false;
|
|
114565
|
+
}
|
|
114566
|
+
let values;
|
|
114567
|
+
if (isSequence) {
|
|
114568
|
+
values = expandSequence(m.body, isAlphaSequence, max, maxLength);
|
|
114569
|
+
}
|
|
114570
|
+
else {
|
|
114571
|
+
let n = parseCommaParts(m.body);
|
|
114572
|
+
if (n.length === 1 && n[0] !== undefined) {
|
|
114573
|
+
// x{{a,b}}y ==> x{a}y x{b}y
|
|
114574
|
+
n = expand_(n[0], max, maxLength, false).map(embrace);
|
|
114575
|
+
//XXX is this necessary? Can't seem to hit it in tests.
|
|
114576
|
+
/* c8 ignore start */
|
|
114577
|
+
if (n.length === 1) {
|
|
114578
|
+
acc = combine(acc, pre + n[0], [''], max, maxLength, dropEmpties && !m.post.length);
|
|
114579
|
+
if (!m.post.length)
|
|
114580
|
+
break;
|
|
114581
|
+
str = m.post;
|
|
114582
|
+
continue;
|
|
114583
|
+
}
|
|
114584
|
+
/* c8 ignore stop */
|
|
114585
|
+
}
|
|
114586
|
+
// Values that `combine` is going to drop as empty produce no result, so
|
|
114587
|
+
// they must not count against `max` - otherwise `{a,,b}` with `max: 2`
|
|
114588
|
+
// would stop at `['a', '']` and yield one result instead of two. Skipping
|
|
114589
|
+
// them outright keeps `values` bounded while leaving `max` a bound on
|
|
114590
|
+
// *kept* results.
|
|
114591
|
+
let dropsEmpties = dropEmpties && !m.post.length && !pre;
|
|
114592
|
+
for (let d = 0; dropsEmpties && d < acc.length; d++) {
|
|
114593
|
+
if (acc[d]) {
|
|
114594
|
+
dropsEmpties = false;
|
|
114595
|
+
}
|
|
114596
|
+
}
|
|
114597
|
+
values = [];
|
|
114598
|
+
let valuesLength = 0;
|
|
114599
|
+
outer: for (let j = 0; j < n.length; j++) {
|
|
114600
|
+
const expanded = expand_(n[j], max, maxLength, false);
|
|
114601
|
+
for (let k = 0; k < expanded.length; k++) {
|
|
114602
|
+
const v = expanded[k];
|
|
114603
|
+
if (dropsEmpties && !v)
|
|
114604
|
+
continue;
|
|
114605
|
+
if (values.length >= max || valuesLength + v.length > maxLength) {
|
|
114606
|
+
break outer;
|
|
114607
|
+
}
|
|
114608
|
+
values.push(v);
|
|
114609
|
+
valuesLength += v.length;
|
|
114610
|
+
}
|
|
114611
|
+
}
|
|
114612
|
+
}
|
|
114613
|
+
acc = combine(acc, pre, values, max, maxLength, dropEmpties && !m.post.length);
|
|
114614
|
+
if (!m.post.length)
|
|
114615
|
+
break;
|
|
114616
|
+
str = m.post;
|
|
114617
|
+
}
|
|
114618
|
+
return acc;
|
|
114619
|
+
}
|
|
114620
|
+
|
|
114621
|
+
const MAX_PATTERN_LENGTH = 1024 * 64;
|
|
114622
|
+
const assertValidPattern = (pattern) => {
|
|
114623
|
+
if (typeof pattern !== 'string') {
|
|
114624
|
+
throw new TypeError('invalid pattern');
|
|
114625
|
+
}
|
|
114626
|
+
if (pattern.length > MAX_PATTERN_LENGTH) {
|
|
114627
|
+
throw new TypeError('pattern is too long');
|
|
114628
|
+
}
|
|
114629
|
+
};
|
|
114630
|
+
|
|
114631
|
+
// translate the various posix character classes into unicode properties
|
|
114632
|
+
// this works across all unicode locales
|
|
114633
|
+
// { <posix class>: [<translation>, /u flag required, negated]
|
|
114634
|
+
const posixClasses = {
|
|
114635
|
+
'[:alnum:]': ['\\p{L}\\p{Nl}\\p{Nd}', true],
|
|
114636
|
+
'[:alpha:]': ['\\p{L}\\p{Nl}', true],
|
|
114637
|
+
'[:ascii:]': ['\\x' + '00-\\x' + '7f', false],
|
|
114638
|
+
'[:blank:]': ['\\p{Zs}\\t', true],
|
|
114639
|
+
'[:cntrl:]': ['\\p{Cc}', true],
|
|
114640
|
+
'[:digit:]': ['\\p{Nd}', true],
|
|
114641
|
+
'[:graph:]': ['\\p{Z}\\p{C}', true, true],
|
|
114642
|
+
'[:lower:]': ['\\p{Ll}', true],
|
|
114643
|
+
'[:print:]': ['\\p{C}', true],
|
|
114644
|
+
'[:punct:]': ['\\p{P}', true],
|
|
114645
|
+
'[:space:]': ['\\p{Z}\\t\\r\\n\\v\\f', true],
|
|
114646
|
+
'[:upper:]': ['\\p{Lu}', true],
|
|
114647
|
+
'[:word:]': ['\\p{L}\\p{Nl}\\p{Nd}\\p{Pc}', true],
|
|
114648
|
+
'[:xdigit:]': ['A-Fa-f0-9', false],
|
|
114649
|
+
};
|
|
114650
|
+
// only need to escape a few things inside of brace expressions
|
|
114651
|
+
// escapes: [ \ ] -
|
|
114652
|
+
const braceEscape = (s) => s.replace(/[[\]\\-]/g, '\\$&');
|
|
114653
|
+
// escape all regexp magic characters
|
|
114654
|
+
const regexpEscape = (s) => s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&');
|
|
114655
|
+
// everything has already been escaped, we just have to join
|
|
114656
|
+
const rangesToString = (ranges) => ranges.join('');
|
|
114657
|
+
// takes a glob string at a posix brace expression, and returns
|
|
114658
|
+
// an equivalent regular expression source, and boolean indicating
|
|
114659
|
+
// whether the /u flag needs to be applied, and the number of chars
|
|
114660
|
+
// consumed to parse the character class.
|
|
114661
|
+
// This also removes out of order ranges, and returns ($.) if the
|
|
114662
|
+
// entire class just no good.
|
|
114663
|
+
const parseClass = (glob, position) => {
|
|
114664
|
+
const pos = position;
|
|
114665
|
+
/* c8 ignore start */
|
|
114666
|
+
if (glob.charAt(pos) !== '[') {
|
|
114667
|
+
throw new Error('not in a brace expression');
|
|
114668
|
+
}
|
|
114669
|
+
/* c8 ignore stop */
|
|
114670
|
+
const ranges = [];
|
|
114671
|
+
const negs = [];
|
|
114672
|
+
let i = pos + 1;
|
|
114673
|
+
let sawStart = false;
|
|
114674
|
+
let uflag = false;
|
|
114675
|
+
let escaping = false;
|
|
114676
|
+
let negate = false;
|
|
114677
|
+
let endPos = pos;
|
|
114678
|
+
let rangeStart = '';
|
|
114679
|
+
WHILE: while (i < glob.length) {
|
|
114680
|
+
const c = glob.charAt(i);
|
|
114681
|
+
if ((c === '!' || c === '^') && i === pos + 1) {
|
|
114682
|
+
negate = true;
|
|
114683
|
+
i++;
|
|
114684
|
+
continue;
|
|
114685
|
+
}
|
|
114686
|
+
if (c === ']' && sawStart && !escaping) {
|
|
114687
|
+
endPos = i + 1;
|
|
114688
|
+
break;
|
|
114689
|
+
}
|
|
114690
|
+
sawStart = true;
|
|
114691
|
+
if (c === '\\') {
|
|
114692
|
+
if (!escaping) {
|
|
114693
|
+
escaping = true;
|
|
114694
|
+
i++;
|
|
114695
|
+
continue;
|
|
114696
|
+
}
|
|
114697
|
+
// escaped \ char, fall through and treat like normal char
|
|
114698
|
+
}
|
|
114699
|
+
if (c === '[' && !escaping) {
|
|
114700
|
+
// either a posix class, a collation equivalent, or just a [
|
|
114701
|
+
for (const [cls, [unip, u, neg]] of Object.entries(posixClasses)) {
|
|
114702
|
+
if (glob.startsWith(cls, i)) {
|
|
114703
|
+
// invalid, [a-[] is fine, but not [a-[:alpha]]
|
|
114704
|
+
if (rangeStart) {
|
|
114705
|
+
return ['$.', false, glob.length - pos, true];
|
|
114706
|
+
}
|
|
114707
|
+
i += cls.length;
|
|
114708
|
+
if (neg)
|
|
114709
|
+
negs.push(unip);
|
|
114710
|
+
else
|
|
114711
|
+
ranges.push(unip);
|
|
114712
|
+
uflag = uflag || u;
|
|
114713
|
+
continue WHILE;
|
|
114714
|
+
}
|
|
114715
|
+
}
|
|
114716
|
+
}
|
|
114717
|
+
// now it's just a normal character, effectively
|
|
114718
|
+
escaping = false;
|
|
114719
|
+
if (rangeStart) {
|
|
114720
|
+
// throw this range away if it's not valid, but others
|
|
114721
|
+
// can still match.
|
|
114722
|
+
if (c > rangeStart) {
|
|
114723
|
+
ranges.push(braceEscape(rangeStart) + '-' + braceEscape(c));
|
|
114724
|
+
}
|
|
114725
|
+
else if (c === rangeStart) {
|
|
114726
|
+
ranges.push(braceEscape(c));
|
|
114727
|
+
}
|
|
114728
|
+
rangeStart = '';
|
|
114729
|
+
i++;
|
|
114730
|
+
continue;
|
|
114731
|
+
}
|
|
114732
|
+
// now might be the start of a range.
|
|
114733
|
+
// can be either c-d or c-] or c<more...>] or c] at this point
|
|
114734
|
+
if (glob.startsWith('-]', i + 1)) {
|
|
114735
|
+
ranges.push(braceEscape(c + '-'));
|
|
114736
|
+
i += 2;
|
|
114737
|
+
continue;
|
|
114738
|
+
}
|
|
114739
|
+
if (glob.startsWith('-', i + 1)) {
|
|
114740
|
+
rangeStart = c;
|
|
114741
|
+
i += 2;
|
|
114742
|
+
continue;
|
|
114743
|
+
}
|
|
114744
|
+
// not the start of a range, just a single character
|
|
114745
|
+
ranges.push(braceEscape(c));
|
|
114746
|
+
i++;
|
|
114747
|
+
}
|
|
114748
|
+
if (endPos < i) {
|
|
114749
|
+
// didn't see the end of the class, not a valid class,
|
|
114750
|
+
// but might still be valid as a literal match.
|
|
114751
|
+
return ['', false, 0, false];
|
|
114752
|
+
}
|
|
114753
|
+
// if we got no ranges and no negates, then we have a range that
|
|
114754
|
+
// cannot possibly match anything, and that poisons the whole glob
|
|
114755
|
+
if (!ranges.length && !negs.length) {
|
|
114756
|
+
return ['$.', false, glob.length - pos, true];
|
|
114757
|
+
}
|
|
114758
|
+
// if we got one positive range, and it's a single character, then that's
|
|
114759
|
+
// not actually a magic pattern, it's just that one literal character.
|
|
114760
|
+
// we should not treat that as "magic", we should just return the literal
|
|
114761
|
+
// character. [_] is a perfectly valid way to escape glob magic chars.
|
|
114762
|
+
if (negs.length === 0 &&
|
|
114763
|
+
ranges.length === 1 &&
|
|
114764
|
+
/^\\?.$/.test(ranges[0]) &&
|
|
114765
|
+
!negate) {
|
|
114766
|
+
const r = ranges[0].length === 2 ? ranges[0].slice(-1) : ranges[0];
|
|
114767
|
+
return [regexpEscape(r), false, endPos - pos, false];
|
|
114768
|
+
}
|
|
114769
|
+
const sranges = '[' + (negate ? '^' : '') + rangesToString(ranges) + ']';
|
|
114770
|
+
const snegs = '[' + (negate ? '' : '^') + rangesToString(negs) + ']';
|
|
114771
|
+
const comb = ranges.length && negs.length ? '(' + sranges + '|' + snegs + ')'
|
|
114772
|
+
: ranges.length ? sranges
|
|
114773
|
+
: snegs;
|
|
114774
|
+
return [comb, uflag, endPos - pos, true];
|
|
114775
|
+
};
|
|
114776
|
+
|
|
114777
|
+
/**
|
|
114778
|
+
* Un-escape a string that has been escaped with {@link escape}.
|
|
114779
|
+
*
|
|
114780
|
+
* If the {@link MinimatchOptions.windowsPathsNoEscape} option is used, then
|
|
114781
|
+
* square-bracket escapes are removed, but not backslash escapes.
|
|
114782
|
+
*
|
|
114783
|
+
* For example, it will turn the string `'[*]'` into `*`, but it will not
|
|
114784
|
+
* turn `'\\*'` into `'*'`, because `\` is a path separator in
|
|
114785
|
+
* `windowsPathsNoEscape` mode.
|
|
114786
|
+
*
|
|
114787
|
+
* When `windowsPathsNoEscape` is not set, then both square-bracket escapes and
|
|
114788
|
+
* backslash escapes are removed.
|
|
114789
|
+
*
|
|
114790
|
+
* Slashes (and backslashes in `windowsPathsNoEscape` mode) cannot be escaped
|
|
114791
|
+
* or unescaped.
|
|
114792
|
+
*
|
|
114793
|
+
* When `magicalBraces` is not set, escapes of braces (`{` and `}`) will not be
|
|
114794
|
+
* unescaped.
|
|
114795
|
+
*/
|
|
114796
|
+
const unescape$1 = (s, { windowsPathsNoEscape = false, magicalBraces = true, } = {}) => {
|
|
114797
|
+
if (magicalBraces) {
|
|
114798
|
+
return windowsPathsNoEscape ?
|
|
114799
|
+
s.replace(/\[([^/\\])\]/g, '$1')
|
|
114800
|
+
: s
|
|
114801
|
+
.replace(/((?!\\).|^)\[([^/\\])\]/g, '$1$2')
|
|
114802
|
+
.replace(/\\([^/])/g, '$1');
|
|
114803
|
+
}
|
|
114804
|
+
return windowsPathsNoEscape ?
|
|
114805
|
+
s.replace(/\[([^/\\{}])\]/g, '$1')
|
|
114806
|
+
: s
|
|
114807
|
+
.replace(/((?!\\).|^)\[([^/\\{}])\]/g, '$1$2')
|
|
114808
|
+
.replace(/\\([^/{}])/g, '$1');
|
|
114809
|
+
};
|
|
114810
|
+
|
|
114811
|
+
// parse a single path portion
|
|
114812
|
+
var _a;
|
|
114813
|
+
const types$2 = new Set(['!', '?', '+', '*', '@']);
|
|
114814
|
+
const isExtglobType = (c) => types$2.has(c);
|
|
114815
|
+
const isExtglobAST = (c) => isExtglobType(c.type);
|
|
114816
|
+
// Map of which extglob types can adopt the children of a nested extglob
|
|
114817
|
+
//
|
|
114818
|
+
// anything but ! can adopt a matching type:
|
|
114819
|
+
// +(a|+(b|c)|d) => +(a|b|c|d)
|
|
114820
|
+
// *(a|*(b|c)|d) => *(a|b|c|d)
|
|
114821
|
+
// @(a|@(b|c)|d) => @(a|b|c|d)
|
|
114822
|
+
// ?(a|?(b|c)|d) => ?(a|b|c|d)
|
|
114823
|
+
//
|
|
114824
|
+
// * can adopt anything, because 0 or repetition is allowed
|
|
114825
|
+
// *(a|?(b|c)|d) => *(a|b|c|d)
|
|
114826
|
+
// *(a|+(b|c)|d) => *(a|b|c|d)
|
|
114827
|
+
// *(a|@(b|c)|d) => *(a|b|c|d)
|
|
114828
|
+
//
|
|
114829
|
+
// + can adopt @, because 1 or repetition is allowed
|
|
114830
|
+
// +(a|@(b|c)|d) => +(a|b|c|d)
|
|
114831
|
+
//
|
|
114832
|
+
// + and @ CANNOT adopt *, because 0 would be allowed
|
|
114833
|
+
// +(a|*(b|c)|d) => would match "", on *(b|c)
|
|
114834
|
+
// @(a|*(b|c)|d) => would match "", on *(b|c)
|
|
114835
|
+
//
|
|
114836
|
+
// + and @ CANNOT adopt ?, because 0 would be allowed
|
|
114837
|
+
// +(a|?(b|c)|d) => would match "", on ?(b|c)
|
|
114838
|
+
// @(a|?(b|c)|d) => would match "", on ?(b|c)
|
|
114839
|
+
//
|
|
114840
|
+
// ? can adopt @, because 0 or 1 is allowed
|
|
114841
|
+
// ?(a|@(b|c)|d) => ?(a|b|c|d)
|
|
114842
|
+
//
|
|
114843
|
+
// ? and @ CANNOT adopt * or +, because >1 would be allowed
|
|
114844
|
+
// ?(a|*(b|c)|d) => would match bbb on *(b|c)
|
|
114845
|
+
// @(a|*(b|c)|d) => would match bbb on *(b|c)
|
|
114846
|
+
// ?(a|+(b|c)|d) => would match bbb on +(b|c)
|
|
114847
|
+
// @(a|+(b|c)|d) => would match bbb on +(b|c)
|
|
114848
|
+
//
|
|
113597
114849
|
// ! CANNOT adopt ! (nothing else can either)
|
|
113598
114850
|
// !(a|!(b|c)|d) => !(a|b|c|d) would fail to match on b (not not b|c)
|
|
113599
114851
|
//
|
|
@@ -120965,1568 +122217,511 @@ class GlobUtil {
|
|
|
120965
122217
|
this.matchSync(m, absolute, ifDir);
|
|
120966
122218
|
}
|
|
120967
122219
|
for (const t of processor.subwalkTargets()) {
|
|
120968
|
-
if (this.maxDepth !== Infinity && t.depth() >= this.maxDepth) {
|
|
120969
|
-
continue;
|
|
120970
|
-
}
|
|
120971
|
-
tasks++;
|
|
120972
|
-
const children = t.readdirSync();
|
|
120973
|
-
this.walkCB3Sync(t, children, processor, next);
|
|
120974
|
-
}
|
|
120975
|
-
next();
|
|
120976
|
-
}
|
|
120977
|
-
walkCB3Sync(target, entries, processor, cb) {
|
|
120978
|
-
processor = processor.filterEntries(target, entries);
|
|
120979
|
-
let tasks = 1;
|
|
120980
|
-
const next = () => {
|
|
120981
|
-
if (--tasks === 0)
|
|
120982
|
-
cb();
|
|
120983
|
-
};
|
|
120984
|
-
for (const [m, absolute, ifDir] of processor.matches.entries()) {
|
|
120985
|
-
if (this.#ignored(m))
|
|
120986
|
-
continue;
|
|
120987
|
-
this.matchSync(m, absolute, ifDir);
|
|
120988
|
-
}
|
|
120989
|
-
for (const [target, patterns] of processor.subwalks.entries()) {
|
|
120990
|
-
tasks++;
|
|
120991
|
-
this.walkCB2Sync(target, patterns, processor.child(), next);
|
|
120992
|
-
}
|
|
120993
|
-
next();
|
|
120994
|
-
}
|
|
120995
|
-
}
|
|
120996
|
-
class GlobWalker extends GlobUtil {
|
|
120997
|
-
matches = new Set();
|
|
120998
|
-
constructor(patterns, path, opts) {
|
|
120999
|
-
super(patterns, path, opts);
|
|
121000
|
-
}
|
|
121001
|
-
matchEmit(e) {
|
|
121002
|
-
this.matches.add(e);
|
|
121003
|
-
}
|
|
121004
|
-
async walk() {
|
|
121005
|
-
if (this.signal?.aborted)
|
|
121006
|
-
throw this.signal.reason;
|
|
121007
|
-
if (this.path.isUnknown()) {
|
|
121008
|
-
await this.path.lstat();
|
|
121009
|
-
}
|
|
121010
|
-
await new Promise((res, rej) => {
|
|
121011
|
-
this.walkCB(this.path, this.patterns, () => {
|
|
121012
|
-
if (this.signal?.aborted) {
|
|
121013
|
-
rej(this.signal.reason);
|
|
121014
|
-
}
|
|
121015
|
-
else {
|
|
121016
|
-
res(this.matches);
|
|
121017
|
-
}
|
|
121018
|
-
});
|
|
121019
|
-
});
|
|
121020
|
-
return this.matches;
|
|
121021
|
-
}
|
|
121022
|
-
walkSync() {
|
|
121023
|
-
if (this.signal?.aborted)
|
|
121024
|
-
throw this.signal.reason;
|
|
121025
|
-
if (this.path.isUnknown()) {
|
|
121026
|
-
this.path.lstatSync();
|
|
121027
|
-
}
|
|
121028
|
-
// nothing for the callback to do, because this never pauses
|
|
121029
|
-
this.walkCBSync(this.path, this.patterns, () => {
|
|
121030
|
-
if (this.signal?.aborted)
|
|
121031
|
-
throw this.signal.reason;
|
|
121032
|
-
});
|
|
121033
|
-
return this.matches;
|
|
121034
|
-
}
|
|
121035
|
-
}
|
|
121036
|
-
class GlobStream extends GlobUtil {
|
|
121037
|
-
results;
|
|
121038
|
-
constructor(patterns, path, opts) {
|
|
121039
|
-
super(patterns, path, opts);
|
|
121040
|
-
this.results = new Minipass({
|
|
121041
|
-
signal: this.signal,
|
|
121042
|
-
objectMode: true,
|
|
121043
|
-
});
|
|
121044
|
-
this.results.on('drain', () => this.resume());
|
|
121045
|
-
this.results.on('resume', () => this.resume());
|
|
121046
|
-
}
|
|
121047
|
-
matchEmit(e) {
|
|
121048
|
-
this.results.write(e);
|
|
121049
|
-
if (!this.results.flowing)
|
|
121050
|
-
this.pause();
|
|
121051
|
-
}
|
|
121052
|
-
stream() {
|
|
121053
|
-
const target = this.path;
|
|
121054
|
-
if (target.isUnknown()) {
|
|
121055
|
-
target.lstat().then(() => {
|
|
121056
|
-
this.walkCB(target, this.patterns, () => this.results.end());
|
|
121057
|
-
});
|
|
121058
|
-
}
|
|
121059
|
-
else {
|
|
121060
|
-
this.walkCB(target, this.patterns, () => this.results.end());
|
|
121061
|
-
}
|
|
121062
|
-
return this.results;
|
|
121063
|
-
}
|
|
121064
|
-
streamSync() {
|
|
121065
|
-
if (this.path.isUnknown()) {
|
|
121066
|
-
this.path.lstatSync();
|
|
121067
|
-
}
|
|
121068
|
-
this.walkCBSync(this.path, this.patterns, () => this.results.end());
|
|
121069
|
-
return this.results;
|
|
121070
|
-
}
|
|
121071
|
-
}
|
|
121072
|
-
|
|
121073
|
-
// if no process global, just call it linux.
|
|
121074
|
-
// so we default to case-sensitive, / separators
|
|
121075
|
-
const defaultPlatform = (typeof process === 'object' &&
|
|
121076
|
-
process &&
|
|
121077
|
-
typeof process.platform === 'string') ?
|
|
121078
|
-
process.platform
|
|
121079
|
-
: 'linux';
|
|
121080
|
-
/**
|
|
121081
|
-
* An object that can perform glob pattern traversals.
|
|
121082
|
-
*/
|
|
121083
|
-
class Glob {
|
|
121084
|
-
absolute;
|
|
121085
|
-
cwd;
|
|
121086
|
-
root;
|
|
121087
|
-
dot;
|
|
121088
|
-
dotRelative;
|
|
121089
|
-
follow;
|
|
121090
|
-
ignore;
|
|
121091
|
-
magicalBraces;
|
|
121092
|
-
mark;
|
|
121093
|
-
matchBase;
|
|
121094
|
-
maxDepth;
|
|
121095
|
-
nobrace;
|
|
121096
|
-
nocase;
|
|
121097
|
-
nodir;
|
|
121098
|
-
noext;
|
|
121099
|
-
noglobstar;
|
|
121100
|
-
pattern;
|
|
121101
|
-
platform;
|
|
121102
|
-
realpath;
|
|
121103
|
-
scurry;
|
|
121104
|
-
stat;
|
|
121105
|
-
signal;
|
|
121106
|
-
windowsPathsNoEscape;
|
|
121107
|
-
withFileTypes;
|
|
121108
|
-
includeChildMatches;
|
|
121109
|
-
/**
|
|
121110
|
-
* The options provided to the constructor.
|
|
121111
|
-
*/
|
|
121112
|
-
opts;
|
|
121113
|
-
/**
|
|
121114
|
-
* An array of parsed immutable {@link Pattern} objects.
|
|
121115
|
-
*/
|
|
121116
|
-
patterns;
|
|
121117
|
-
/**
|
|
121118
|
-
* All options are stored as properties on the `Glob` object.
|
|
121119
|
-
*
|
|
121120
|
-
* See {@link GlobOptions} for full options descriptions.
|
|
121121
|
-
*
|
|
121122
|
-
* Note that a previous `Glob` object can be passed as the
|
|
121123
|
-
* `GlobOptions` to another `Glob` instantiation to re-use settings
|
|
121124
|
-
* and caches with a new pattern.
|
|
121125
|
-
*
|
|
121126
|
-
* Traversal functions can be called multiple times to run the walk
|
|
121127
|
-
* again.
|
|
121128
|
-
*/
|
|
121129
|
-
constructor(pattern, opts) {
|
|
121130
|
-
/* c8 ignore start */
|
|
121131
|
-
if (!opts)
|
|
121132
|
-
throw new TypeError('glob options required');
|
|
121133
|
-
/* c8 ignore stop */
|
|
121134
|
-
this.withFileTypes = !!opts.withFileTypes;
|
|
121135
|
-
this.signal = opts.signal;
|
|
121136
|
-
this.follow = !!opts.follow;
|
|
121137
|
-
this.dot = !!opts.dot;
|
|
121138
|
-
this.dotRelative = !!opts.dotRelative;
|
|
121139
|
-
this.nodir = !!opts.nodir;
|
|
121140
|
-
this.mark = !!opts.mark;
|
|
121141
|
-
if (!opts.cwd) {
|
|
121142
|
-
this.cwd = '';
|
|
121143
|
-
}
|
|
121144
|
-
else if (opts.cwd instanceof URL || opts.cwd.startsWith('file://')) {
|
|
121145
|
-
opts.cwd = require$$0$l.fileURLToPath(opts.cwd);
|
|
121146
|
-
}
|
|
121147
|
-
this.cwd = opts.cwd || '';
|
|
121148
|
-
this.root = opts.root;
|
|
121149
|
-
this.magicalBraces = !!opts.magicalBraces;
|
|
121150
|
-
this.nobrace = !!opts.nobrace;
|
|
121151
|
-
this.noext = !!opts.noext;
|
|
121152
|
-
this.realpath = !!opts.realpath;
|
|
121153
|
-
this.absolute = opts.absolute;
|
|
121154
|
-
this.includeChildMatches = opts.includeChildMatches !== false;
|
|
121155
|
-
this.noglobstar = !!opts.noglobstar;
|
|
121156
|
-
this.matchBase = !!opts.matchBase;
|
|
121157
|
-
this.maxDepth =
|
|
121158
|
-
typeof opts.maxDepth === 'number' ? opts.maxDepth : Infinity;
|
|
121159
|
-
this.stat = !!opts.stat;
|
|
121160
|
-
this.ignore = opts.ignore;
|
|
121161
|
-
if (this.withFileTypes && this.absolute !== undefined) {
|
|
121162
|
-
throw new Error('cannot set absolute and withFileTypes:true');
|
|
121163
|
-
}
|
|
121164
|
-
if (typeof pattern === 'string') {
|
|
121165
|
-
pattern = [pattern];
|
|
121166
|
-
}
|
|
121167
|
-
this.windowsPathsNoEscape =
|
|
121168
|
-
!!opts.windowsPathsNoEscape ||
|
|
121169
|
-
opts.allowWindowsEscape ===
|
|
121170
|
-
false;
|
|
121171
|
-
if (this.windowsPathsNoEscape) {
|
|
121172
|
-
pattern = pattern.map(p => p.replace(/\\/g, '/'));
|
|
121173
|
-
}
|
|
121174
|
-
if (this.matchBase) {
|
|
121175
|
-
if (opts.noglobstar) {
|
|
121176
|
-
throw new TypeError('base matching requires globstar');
|
|
121177
|
-
}
|
|
121178
|
-
pattern = pattern.map(p => (p.includes('/') ? p : `./**/${p}`));
|
|
121179
|
-
}
|
|
121180
|
-
this.pattern = pattern;
|
|
121181
|
-
this.platform = opts.platform || defaultPlatform;
|
|
121182
|
-
this.opts = { ...opts, platform: this.platform };
|
|
121183
|
-
if (opts.scurry) {
|
|
121184
|
-
this.scurry = opts.scurry;
|
|
121185
|
-
if (opts.nocase !== undefined &&
|
|
121186
|
-
opts.nocase !== opts.scurry.nocase) {
|
|
121187
|
-
throw new Error('nocase option contradicts provided scurry option');
|
|
121188
|
-
}
|
|
121189
|
-
}
|
|
121190
|
-
else {
|
|
121191
|
-
const Scurry = opts.platform === 'win32' ? PathScurryWin32
|
|
121192
|
-
: opts.platform === 'darwin' ? PathScurryDarwin
|
|
121193
|
-
: opts.platform ? PathScurryPosix
|
|
121194
|
-
: PathScurry;
|
|
121195
|
-
this.scurry = new Scurry(this.cwd, {
|
|
121196
|
-
nocase: opts.nocase,
|
|
121197
|
-
fs: opts.fs,
|
|
121198
|
-
});
|
|
121199
|
-
}
|
|
121200
|
-
this.nocase = this.scurry.nocase;
|
|
121201
|
-
// If you do nocase:true on a case-sensitive file system, then
|
|
121202
|
-
// we need to use regexps instead of strings for non-magic
|
|
121203
|
-
// path portions, because statting `aBc` won't return results
|
|
121204
|
-
// for the file `AbC` for example.
|
|
121205
|
-
const nocaseMagicOnly = this.platform === 'darwin' || this.platform === 'win32';
|
|
121206
|
-
const mmo = {
|
|
121207
|
-
// default nocase based on platform
|
|
121208
|
-
...opts,
|
|
121209
|
-
dot: this.dot,
|
|
121210
|
-
matchBase: this.matchBase,
|
|
121211
|
-
nobrace: this.nobrace,
|
|
121212
|
-
nocase: this.nocase,
|
|
121213
|
-
nocaseMagicOnly,
|
|
121214
|
-
nocomment: true,
|
|
121215
|
-
noext: this.noext,
|
|
121216
|
-
nonegate: true,
|
|
121217
|
-
optimizationLevel: 2,
|
|
121218
|
-
platform: this.platform,
|
|
121219
|
-
windowsPathsNoEscape: this.windowsPathsNoEscape,
|
|
121220
|
-
debug: !!this.opts.debug,
|
|
121221
|
-
};
|
|
121222
|
-
const mms = this.pattern.map(p => new Minimatch(p, mmo));
|
|
121223
|
-
const [matchSet, globParts] = mms.reduce((set, m) => {
|
|
121224
|
-
set[0].push(...m.set);
|
|
121225
|
-
set[1].push(...m.globParts);
|
|
121226
|
-
return set;
|
|
121227
|
-
}, [[], []]);
|
|
121228
|
-
this.patterns = matchSet.map((set, i) => {
|
|
121229
|
-
const g = globParts[i];
|
|
121230
|
-
/* c8 ignore start */
|
|
121231
|
-
if (!g)
|
|
121232
|
-
throw new Error('invalid pattern object');
|
|
121233
|
-
/* c8 ignore stop */
|
|
121234
|
-
return new Pattern(set, g, 0, this.platform);
|
|
121235
|
-
});
|
|
121236
|
-
}
|
|
121237
|
-
async walk() {
|
|
121238
|
-
// Walkers always return array of Path objects, so we just have to
|
|
121239
|
-
// coerce them into the right shape. It will have already called
|
|
121240
|
-
// realpath() if the option was set to do so, so we know that's cached.
|
|
121241
|
-
// start out knowing the cwd, at least
|
|
121242
|
-
return [
|
|
121243
|
-
...(await new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
121244
|
-
...this.opts,
|
|
121245
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121246
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121247
|
-
: Infinity,
|
|
121248
|
-
platform: this.platform,
|
|
121249
|
-
nocase: this.nocase,
|
|
121250
|
-
includeChildMatches: this.includeChildMatches,
|
|
121251
|
-
}).walk()),
|
|
121252
|
-
];
|
|
121253
|
-
}
|
|
121254
|
-
walkSync() {
|
|
121255
|
-
return [
|
|
121256
|
-
...new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
121257
|
-
...this.opts,
|
|
121258
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121259
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121260
|
-
: Infinity,
|
|
121261
|
-
platform: this.platform,
|
|
121262
|
-
nocase: this.nocase,
|
|
121263
|
-
includeChildMatches: this.includeChildMatches,
|
|
121264
|
-
}).walkSync(),
|
|
121265
|
-
];
|
|
121266
|
-
}
|
|
121267
|
-
stream() {
|
|
121268
|
-
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
121269
|
-
...this.opts,
|
|
121270
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121271
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121272
|
-
: Infinity,
|
|
121273
|
-
platform: this.platform,
|
|
121274
|
-
nocase: this.nocase,
|
|
121275
|
-
includeChildMatches: this.includeChildMatches,
|
|
121276
|
-
}).stream();
|
|
121277
|
-
}
|
|
121278
|
-
streamSync() {
|
|
121279
|
-
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
121280
|
-
...this.opts,
|
|
121281
|
-
maxDepth: this.maxDepth !== Infinity ?
|
|
121282
|
-
this.maxDepth + this.scurry.cwd.depth()
|
|
121283
|
-
: Infinity,
|
|
121284
|
-
platform: this.platform,
|
|
121285
|
-
nocase: this.nocase,
|
|
121286
|
-
includeChildMatches: this.includeChildMatches,
|
|
121287
|
-
}).streamSync();
|
|
121288
|
-
}
|
|
121289
|
-
/**
|
|
121290
|
-
* Default sync iteration function. Returns a Generator that
|
|
121291
|
-
* iterates over the results.
|
|
121292
|
-
*/
|
|
121293
|
-
iterateSync() {
|
|
121294
|
-
return this.streamSync()[Symbol.iterator]();
|
|
121295
|
-
}
|
|
121296
|
-
[Symbol.iterator]() {
|
|
121297
|
-
return this.iterateSync();
|
|
121298
|
-
}
|
|
121299
|
-
/**
|
|
121300
|
-
* Default async iteration function. Returns an AsyncGenerator that
|
|
121301
|
-
* iterates over the results.
|
|
121302
|
-
*/
|
|
121303
|
-
iterate() {
|
|
121304
|
-
return this.stream()[Symbol.asyncIterator]();
|
|
121305
|
-
}
|
|
121306
|
-
[Symbol.asyncIterator]() {
|
|
121307
|
-
return this.iterate();
|
|
121308
|
-
}
|
|
121309
|
-
}
|
|
121310
|
-
|
|
121311
|
-
/**
|
|
121312
|
-
* Return true if the patterns provided contain any magic glob characters,
|
|
121313
|
-
* given the options provided.
|
|
121314
|
-
*
|
|
121315
|
-
* Brace expansion is not considered "magic" unless the `magicalBraces` option
|
|
121316
|
-
* is set, as brace expansion just turns one string into an array of strings.
|
|
121317
|
-
* So a pattern like `'x{a,b}y'` would return `false`, because `'xay'` and
|
|
121318
|
-
* `'xby'` both do not contain any magic glob characters, and it's treated the
|
|
121319
|
-
* same as if you had called it on `['xay', 'xby']`. When `magicalBraces:true`
|
|
121320
|
-
* is in the options, brace expansion _is_ treated as a pattern having magic.
|
|
121321
|
-
*/
|
|
121322
|
-
const hasMagic = (pattern, options = {}) => {
|
|
121323
|
-
if (!Array.isArray(pattern)) {
|
|
121324
|
-
pattern = [pattern];
|
|
121325
|
-
}
|
|
121326
|
-
for (const p of pattern) {
|
|
121327
|
-
if (new Minimatch(p, options).hasMagic())
|
|
121328
|
-
return true;
|
|
121329
|
-
}
|
|
121330
|
-
return false;
|
|
121331
|
-
};
|
|
121332
|
-
|
|
121333
|
-
function globStreamSync(pattern, options = {}) {
|
|
121334
|
-
return new Glob(pattern, options).streamSync();
|
|
121335
|
-
}
|
|
121336
|
-
function globStream(pattern, options = {}) {
|
|
121337
|
-
return new Glob(pattern, options).stream();
|
|
121338
|
-
}
|
|
121339
|
-
function globSync(pattern, options = {}) {
|
|
121340
|
-
return new Glob(pattern, options).walkSync();
|
|
121341
|
-
}
|
|
121342
|
-
async function glob_(pattern, options = {}) {
|
|
121343
|
-
return new Glob(pattern, options).walk();
|
|
121344
|
-
}
|
|
121345
|
-
function globIterateSync(pattern, options = {}) {
|
|
121346
|
-
return new Glob(pattern, options).iterateSync();
|
|
121347
|
-
}
|
|
121348
|
-
function globIterate(pattern, options = {}) {
|
|
121349
|
-
return new Glob(pattern, options).iterate();
|
|
121350
|
-
}
|
|
121351
|
-
// aliases: glob.sync.stream() glob.stream.sync() glob.sync() etc
|
|
121352
|
-
const streamSync = globStreamSync;
|
|
121353
|
-
const stream = Object.assign(globStream, { sync: globStreamSync });
|
|
121354
|
-
const iterateSync = globIterateSync;
|
|
121355
|
-
const iterate = Object.assign(globIterate, {
|
|
121356
|
-
sync: globIterateSync,
|
|
121357
|
-
});
|
|
121358
|
-
const sync = Object.assign(globSync, {
|
|
121359
|
-
stream: globStreamSync,
|
|
121360
|
-
iterate: globIterateSync,
|
|
121361
|
-
});
|
|
121362
|
-
const glob = Object.assign(glob_, {
|
|
121363
|
-
glob: glob_,
|
|
121364
|
-
globSync,
|
|
121365
|
-
sync,
|
|
121366
|
-
globStream,
|
|
121367
|
-
stream,
|
|
121368
|
-
globStreamSync,
|
|
121369
|
-
streamSync,
|
|
121370
|
-
globIterate,
|
|
121371
|
-
iterate,
|
|
121372
|
-
globIterateSync,
|
|
121373
|
-
iterateSync,
|
|
121374
|
-
Glob,
|
|
121375
|
-
hasMagic,
|
|
121376
|
-
escape: escape$1,
|
|
121377
|
-
unescape: unescape$1,
|
|
121378
|
-
});
|
|
121379
|
-
glob.glob = glob;
|
|
121380
|
-
|
|
121381
|
-
function matchesQuantizationVariant({ filePath, variant }) {
|
|
121382
|
-
if (!variant) {
|
|
121383
|
-
return false;
|
|
121384
|
-
}
|
|
121385
|
-
const escapedVariant = variant.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
121386
|
-
const matcher = new RegExp(`(^|[\\-./_])${escapedVariant}(?=[\\-./]|$)`, "i");
|
|
121387
|
-
const normalizedPath = filePath.replace(/\\/g, "/");
|
|
121388
|
-
const segments = normalizedPath.split("/").filter(Boolean);
|
|
121389
|
-
if (segments.length === 0) {
|
|
121390
|
-
return false;
|
|
121391
|
-
}
|
|
121392
|
-
const filename = segments[segments.length - 1].replace(/\.gguf$/i, "");
|
|
121393
|
-
if (matcher.test(filename)) {
|
|
121394
|
-
return true;
|
|
121395
|
-
}
|
|
121396
|
-
return segments.slice(0, -1).some(segment => matcher.test(segment));
|
|
121397
|
-
}
|
|
121398
|
-
async function findQuantizedModelTarget({ model, path }) {
|
|
121399
|
-
if (model.source.type === "storage") {
|
|
121400
|
-
throw new Error("Model storage not supported yet");
|
|
121401
|
-
}
|
|
121402
|
-
if (model.format !== "gguf") {
|
|
121403
|
-
throw new Error(`Model format not supported: ${model.format}`);
|
|
121404
|
-
}
|
|
121405
|
-
const [, variant = null] = model.source.slug.split(":");
|
|
121406
|
-
const modelFiles = (await glob("**/*.gguf", {
|
|
121407
|
-
absolute: true,
|
|
121408
|
-
cwd: path,
|
|
121409
|
-
nodir: true
|
|
121410
|
-
})).filter(file => !/(mmproj|clip)/i.test(file));
|
|
121411
|
-
if (modelFiles.length <= 0) {
|
|
121412
|
-
throw new Error(`No models found for format: ${model.format}`);
|
|
121413
|
-
}
|
|
121414
|
-
modelFiles.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: "base" }));
|
|
121415
|
-
if (!variant) {
|
|
121416
|
-
return modelFiles[0];
|
|
121417
|
-
}
|
|
121418
|
-
const matches = modelFiles.filter(fileName => matchesQuantizationVariant({ filePath: fileName, variant: variant ?? "" }));
|
|
121419
|
-
if (matches.length === 0) {
|
|
121420
|
-
throw new Error(`No model found for format and variant: ${model.format} / ${variant}`);
|
|
121421
|
-
}
|
|
121422
|
-
return matches[0];
|
|
121423
|
-
}
|
|
121424
|
-
|
|
121425
|
-
const VLLM_START_ARGS = ["-m", "vllm.entrypoints.openai.api_server", "--host", "0.0.0.0"];
|
|
121426
|
-
const VLLM_EXECUTABLE = "python3";
|
|
121427
|
-
const DEFAULT_VLLM_CONTEXT_LENGTH = 2048;
|
|
121428
|
-
async function startVLLM({ enginePort, targetDirectory }) {
|
|
121429
|
-
const contextLength = Math.max(1, this.contextLength ?? DEFAULT_VLLM_CONTEXT_LENGTH);
|
|
121430
|
-
let modelPath = targetDirectory;
|
|
121431
|
-
if (this.model.format === "gguf") {
|
|
121432
|
-
modelPath = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
|
|
121433
|
-
}
|
|
121434
|
-
const engineConfig = this.engineConfig;
|
|
121435
|
-
const device = typeof engineConfig?.device === "string" ? engineConfig.device : process.env.VLLM_DEVICE;
|
|
121436
|
-
const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : process.env.VLLM_DTYPE;
|
|
121437
|
-
const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
|
|
121438
|
-
const args = [
|
|
121439
|
-
...VLLM_START_ARGS,
|
|
121440
|
-
"--port",
|
|
121441
|
-
String(enginePort),
|
|
121442
|
-
"--model",
|
|
121443
|
-
modelPath,
|
|
121444
|
-
"--served-model-name",
|
|
121445
|
-
SERVED_MODEL_NAME,
|
|
121446
|
-
"--max-model-len",
|
|
121447
|
-
String(contextLength),
|
|
121448
|
-
"--tensor-parallel-size",
|
|
121449
|
-
String(tensorParallelSize)
|
|
121450
|
-
];
|
|
121451
|
-
if (this.model.taskType === "embeddings") {
|
|
121452
|
-
args.push("--task", "embed");
|
|
121453
|
-
}
|
|
121454
|
-
if (device) {
|
|
121455
|
-
args.push("--device", device);
|
|
121456
|
-
}
|
|
121457
|
-
if (dtype) {
|
|
121458
|
-
args.push("--dtype", dtype);
|
|
121459
|
-
}
|
|
121460
|
-
args.push(...parseExtraArgs(engineConfig?.extraArgs));
|
|
121461
|
-
if (this.model.multimodalEnabled) {
|
|
121462
|
-
args.push("--limit-mm-per-prompt", process.env.VLLM_MM_LIMIT ?? '{"image":5}');
|
|
121463
|
-
}
|
|
121464
|
-
if (process.env.VLLM_TRUST_REMOTE_CODE === "true") {
|
|
121465
|
-
args.push("--trust-remote-code");
|
|
121466
|
-
}
|
|
121467
|
-
return createEngineProcess({ args, bin: VLLM_EXECUTABLE, logger: this.logger });
|
|
121468
|
-
}
|
|
121469
|
-
|
|
121470
|
-
// src/lib/cache-management.ts
|
|
121471
|
-
|
|
121472
|
-
// src/consts.ts
|
|
121473
|
-
var HUB_URL = "https://huggingface.co";
|
|
121474
|
-
|
|
121475
|
-
// src/error.ts
|
|
121476
|
-
async function createApiError(response, opts) {
|
|
121477
|
-
const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
|
|
121478
|
-
error.message = `Api error with status ${error.statusCode}${""}`;
|
|
121479
|
-
const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". ");
|
|
121480
|
-
if (response.headers.get("Content-Type")?.startsWith("application/json")) {
|
|
121481
|
-
const json = await response.json();
|
|
121482
|
-
error.message = json.error || json.message || error.message;
|
|
121483
|
-
if (json.error_description) {
|
|
121484
|
-
error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description;
|
|
121485
|
-
}
|
|
121486
|
-
error.data = json;
|
|
121487
|
-
} else {
|
|
121488
|
-
error.data = { message: await response.text() };
|
|
121489
|
-
}
|
|
121490
|
-
error.message += `. ${trailer}`;
|
|
121491
|
-
throw error;
|
|
121492
|
-
}
|
|
121493
|
-
var HubApiError = class extends Error {
|
|
121494
|
-
statusCode;
|
|
121495
|
-
url;
|
|
121496
|
-
requestId;
|
|
121497
|
-
data;
|
|
121498
|
-
constructor(url, statusCode, requestId, message) {
|
|
121499
|
-
super(message);
|
|
121500
|
-
this.statusCode = statusCode;
|
|
121501
|
-
this.requestId = requestId;
|
|
121502
|
-
this.url = url;
|
|
121503
|
-
}
|
|
121504
|
-
};
|
|
121505
|
-
var InvalidApiResponseFormatError = class extends Error {
|
|
121506
|
-
};
|
|
121507
|
-
|
|
121508
|
-
// src/utils/checkCredentials.ts
|
|
121509
|
-
function checkAccessToken(accessToken) {
|
|
121510
|
-
if (!accessToken.startsWith("hf_")) {
|
|
121511
|
-
throw new TypeError("Your access token must start with 'hf_'");
|
|
121512
|
-
}
|
|
121513
|
-
}
|
|
121514
|
-
function checkCredentials(params) {
|
|
121515
|
-
if (params.accessToken) {
|
|
121516
|
-
checkAccessToken(params.accessToken);
|
|
121517
|
-
return params.accessToken;
|
|
121518
|
-
}
|
|
121519
|
-
if (params.credentials?.accessToken) {
|
|
121520
|
-
checkAccessToken(params.credentials.accessToken);
|
|
121521
|
-
return params.credentials.accessToken;
|
|
121522
|
-
}
|
|
121523
|
-
}
|
|
121524
|
-
|
|
121525
|
-
// src/utils/toRepoId.ts
|
|
121526
|
-
function toRepoId(repo) {
|
|
121527
|
-
if (typeof repo !== "string") {
|
|
121528
|
-
return repo;
|
|
121529
|
-
}
|
|
121530
|
-
if (repo.startsWith("model/") || repo.startsWith("models/")) {
|
|
121531
|
-
throw new TypeError(
|
|
121532
|
-
"A repo designation for a model should not start with 'models/', directly specify the model namespace / name"
|
|
121533
|
-
);
|
|
121534
|
-
}
|
|
121535
|
-
if (repo.startsWith("space/")) {
|
|
121536
|
-
throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'");
|
|
121537
|
-
}
|
|
121538
|
-
if (repo.startsWith("dataset/")) {
|
|
121539
|
-
throw new TypeError("Datasets should start with 'dataset/', plural, not 'dataset/'");
|
|
121540
|
-
}
|
|
121541
|
-
const slashes = repo.split("/").length - 1;
|
|
121542
|
-
if (repo.startsWith("spaces/")) {
|
|
121543
|
-
if (slashes !== 2) {
|
|
121544
|
-
throw new TypeError("Space Id must include namespace and name of the space");
|
|
121545
|
-
}
|
|
121546
|
-
return {
|
|
121547
|
-
type: "space",
|
|
121548
|
-
name: repo.slice("spaces/".length)
|
|
121549
|
-
};
|
|
121550
|
-
}
|
|
121551
|
-
if (repo.startsWith("datasets/")) {
|
|
121552
|
-
if (slashes > 2) {
|
|
121553
|
-
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
121554
|
-
}
|
|
121555
|
-
return {
|
|
121556
|
-
type: "dataset",
|
|
121557
|
-
name: repo.slice("datasets/".length)
|
|
121558
|
-
};
|
|
121559
|
-
}
|
|
121560
|
-
if (slashes > 1) {
|
|
121561
|
-
throw new TypeError("Too many slashes in repo designation: " + repo);
|
|
121562
|
-
}
|
|
121563
|
-
return {
|
|
121564
|
-
type: "model",
|
|
121565
|
-
name: repo
|
|
121566
|
-
};
|
|
121567
|
-
}
|
|
121568
|
-
new Promise((r) => {
|
|
121569
|
-
});
|
|
121570
|
-
|
|
121571
|
-
// src/utils/combineUint8Arrays.ts
|
|
121572
|
-
function combineUint8Arrays(a, b) {
|
|
121573
|
-
const aLength = a.length;
|
|
121574
|
-
const combinedBytes = new Uint8Array(aLength + b.length);
|
|
121575
|
-
combinedBytes.set(a);
|
|
121576
|
-
combinedBytes.set(b, aLength);
|
|
121577
|
-
return combinedBytes;
|
|
121578
|
-
}
|
|
121579
|
-
function readU64(b, n) {
|
|
121580
|
-
let x = 0;
|
|
121581
|
-
x |= b[n++] << 0;
|
|
121582
|
-
x |= b[n++] << 8;
|
|
121583
|
-
x |= b[n++] << 16;
|
|
121584
|
-
x |= b[n++] << 24;
|
|
121585
|
-
x |= b[n++] << 32;
|
|
121586
|
-
x |= b[n++] << 40;
|
|
121587
|
-
x |= b[n++] << 48;
|
|
121588
|
-
x |= b[n++] << 56;
|
|
121589
|
-
return x;
|
|
121590
|
-
}
|
|
121591
|
-
function readU32(b, n) {
|
|
121592
|
-
let x = 0;
|
|
121593
|
-
x |= b[n++] << 0;
|
|
121594
|
-
x |= b[n++] << 8;
|
|
121595
|
-
x |= b[n++] << 16;
|
|
121596
|
-
x |= b[n++] << 24;
|
|
121597
|
-
return x;
|
|
121598
|
-
}
|
|
121599
|
-
|
|
121600
|
-
// src/vendor/lz4js/index.ts
|
|
121601
|
-
var minMatch = 4;
|
|
121602
|
-
var hashSize = 1 << 16;
|
|
121603
|
-
makeHashTable();
|
|
121604
|
-
var magicNum = 407708164;
|
|
121605
|
-
var fdContentChksum = 4;
|
|
121606
|
-
var fdContentSize = 8;
|
|
121607
|
-
var fdBlockChksum = 16;
|
|
121608
|
-
var fdVersion = 64;
|
|
121609
|
-
var fdVersionMask = 192;
|
|
121610
|
-
var bsUncompressed = 2147483648;
|
|
121611
|
-
var bsShift = 4;
|
|
121612
|
-
var bsMask = 7;
|
|
121613
|
-
var bsMap = {
|
|
121614
|
-
4: 65536,
|
|
121615
|
-
5: 262144,
|
|
121616
|
-
6: 1048576,
|
|
121617
|
-
7: 4194304
|
|
121618
|
-
};
|
|
121619
|
-
function makeHashTable() {
|
|
121620
|
-
try {
|
|
121621
|
-
return new Uint32Array(hashSize);
|
|
121622
|
-
} catch (error) {
|
|
121623
|
-
const hashTable2 = new Array(hashSize);
|
|
121624
|
-
for (let i = 0; i < hashSize; i++) {
|
|
121625
|
-
hashTable2[i] = 0;
|
|
121626
|
-
}
|
|
121627
|
-
return hashTable2;
|
|
121628
|
-
}
|
|
121629
|
-
}
|
|
121630
|
-
function makeBuffer(size) {
|
|
121631
|
-
return new Uint8Array(size);
|
|
121632
|
-
}
|
|
121633
|
-
function sliceArray(array, start, end) {
|
|
121634
|
-
return array.slice(start, end);
|
|
121635
|
-
}
|
|
121636
|
-
function decompressBound(src) {
|
|
121637
|
-
let sIndex = 0;
|
|
121638
|
-
if (readU32(src, sIndex) !== magicNum) {
|
|
121639
|
-
throw new Error("invalid magic number");
|
|
121640
|
-
}
|
|
121641
|
-
sIndex += 4;
|
|
121642
|
-
const descriptor = src[sIndex++];
|
|
121643
|
-
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
121644
|
-
throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask));
|
|
121645
|
-
}
|
|
121646
|
-
const useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
121647
|
-
const useContentSize = (descriptor & fdContentSize) !== 0;
|
|
121648
|
-
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
121649
|
-
if (bsMap[bsIdx] === void 0) {
|
|
121650
|
-
throw new Error("invalid block size " + bsIdx);
|
|
121651
|
-
}
|
|
121652
|
-
const maxBlockSize = bsMap[bsIdx];
|
|
121653
|
-
if (useContentSize) {
|
|
121654
|
-
return readU64(src, sIndex);
|
|
121655
|
-
}
|
|
121656
|
-
sIndex++;
|
|
121657
|
-
let maxSize = 0;
|
|
121658
|
-
while (true) {
|
|
121659
|
-
let blockSize = readU32(src, sIndex);
|
|
121660
|
-
sIndex += 4;
|
|
121661
|
-
if (blockSize & bsUncompressed) {
|
|
121662
|
-
blockSize &= ~bsUncompressed;
|
|
121663
|
-
maxSize += blockSize;
|
|
121664
|
-
} else if (blockSize > 0) {
|
|
121665
|
-
maxSize += maxBlockSize;
|
|
121666
|
-
}
|
|
121667
|
-
if (blockSize === 0) {
|
|
121668
|
-
return maxSize;
|
|
121669
|
-
}
|
|
121670
|
-
if (useBlockSum) {
|
|
121671
|
-
sIndex += 4;
|
|
121672
|
-
}
|
|
121673
|
-
sIndex += blockSize;
|
|
121674
|
-
}
|
|
121675
|
-
}
|
|
121676
|
-
function decompressBlock(src, dst, sIndex, sLength, dIndex) {
|
|
121677
|
-
let mLength, mOffset, sEnd, n, i;
|
|
121678
|
-
const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0;
|
|
121679
|
-
sEnd = sIndex + sLength;
|
|
121680
|
-
while (sIndex < sEnd) {
|
|
121681
|
-
const token = src[sIndex++];
|
|
121682
|
-
let literalCount = token >> 4;
|
|
121683
|
-
if (literalCount > 0) {
|
|
121684
|
-
if (literalCount === 15) {
|
|
121685
|
-
while (true) {
|
|
121686
|
-
literalCount += src[sIndex];
|
|
121687
|
-
if (src[sIndex++] !== 255) {
|
|
121688
|
-
break;
|
|
121689
|
-
}
|
|
121690
|
-
}
|
|
121691
|
-
}
|
|
121692
|
-
for (n = sIndex + literalCount; sIndex < n; ) {
|
|
121693
|
-
dst[dIndex++] = src[sIndex++];
|
|
121694
|
-
}
|
|
121695
|
-
}
|
|
121696
|
-
if (sIndex >= sEnd) {
|
|
121697
|
-
break;
|
|
121698
|
-
}
|
|
121699
|
-
mLength = token & 15;
|
|
121700
|
-
mOffset = src[sIndex++] | src[sIndex++] << 8;
|
|
121701
|
-
if (mLength === 15) {
|
|
121702
|
-
while (true) {
|
|
121703
|
-
mLength += src[sIndex];
|
|
121704
|
-
if (src[sIndex++] !== 255) {
|
|
121705
|
-
break;
|
|
121706
|
-
}
|
|
121707
|
-
}
|
|
121708
|
-
}
|
|
121709
|
-
mLength += minMatch;
|
|
121710
|
-
if (hasCopyWithin && mOffset === 1) {
|
|
121711
|
-
dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength);
|
|
121712
|
-
dIndex += mLength;
|
|
121713
|
-
} else if (hasCopyWithin && mOffset > mLength && mLength > 31) {
|
|
121714
|
-
dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength);
|
|
121715
|
-
dIndex += mLength;
|
|
121716
|
-
} else {
|
|
121717
|
-
for (i = dIndex - mOffset, n = i + mLength; i < n; ) {
|
|
121718
|
-
dst[dIndex++] = dst[i++] | 0;
|
|
121719
|
-
}
|
|
121720
|
-
}
|
|
121721
|
-
}
|
|
121722
|
-
return dIndex;
|
|
121723
|
-
}
|
|
121724
|
-
function decompressFrame(src, dst) {
|
|
121725
|
-
let useBlockSum, useContentSum, useContentSize, descriptor;
|
|
121726
|
-
let sIndex = 0;
|
|
121727
|
-
let dIndex = 0;
|
|
121728
|
-
if (readU32(src, sIndex) !== magicNum) {
|
|
121729
|
-
throw new Error("invalid magic number");
|
|
121730
|
-
}
|
|
121731
|
-
sIndex += 4;
|
|
121732
|
-
descriptor = src[sIndex++];
|
|
121733
|
-
if ((descriptor & fdVersionMask) !== fdVersion) {
|
|
121734
|
-
throw new Error("incompatible descriptor version");
|
|
121735
|
-
}
|
|
121736
|
-
useBlockSum = (descriptor & fdBlockChksum) !== 0;
|
|
121737
|
-
useContentSum = (descriptor & fdContentChksum) !== 0;
|
|
121738
|
-
useContentSize = (descriptor & fdContentSize) !== 0;
|
|
121739
|
-
const bsIdx = src[sIndex++] >> bsShift & bsMask;
|
|
121740
|
-
if (bsMap[bsIdx] === void 0) {
|
|
121741
|
-
throw new Error("invalid block size");
|
|
121742
|
-
}
|
|
121743
|
-
if (useContentSize) {
|
|
121744
|
-
sIndex += 8;
|
|
121745
|
-
}
|
|
121746
|
-
sIndex++;
|
|
121747
|
-
while (true) {
|
|
121748
|
-
var compSize;
|
|
121749
|
-
compSize = readU32(src, sIndex);
|
|
121750
|
-
sIndex += 4;
|
|
121751
|
-
if (compSize === 0) {
|
|
121752
|
-
break;
|
|
121753
|
-
}
|
|
121754
|
-
if (useBlockSum) {
|
|
121755
|
-
sIndex += 4;
|
|
121756
|
-
}
|
|
121757
|
-
if ((compSize & bsUncompressed) !== 0) {
|
|
121758
|
-
compSize &= ~bsUncompressed;
|
|
121759
|
-
for (let j = 0; j < compSize; j++) {
|
|
121760
|
-
dst[dIndex++] = src[sIndex++];
|
|
121761
|
-
}
|
|
121762
|
-
} else {
|
|
121763
|
-
dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex);
|
|
121764
|
-
sIndex += compSize;
|
|
121765
|
-
}
|
|
121766
|
-
}
|
|
121767
|
-
if (useContentSum) {
|
|
121768
|
-
sIndex += 4;
|
|
121769
|
-
}
|
|
121770
|
-
return dIndex;
|
|
121771
|
-
}
|
|
121772
|
-
function decompress(src, maxSize) {
|
|
121773
|
-
let dst, size;
|
|
121774
|
-
if (maxSize === void 0) {
|
|
121775
|
-
maxSize = decompressBound(src);
|
|
121776
|
-
}
|
|
121777
|
-
dst = makeBuffer(maxSize);
|
|
121778
|
-
size = decompressFrame(src, dst);
|
|
121779
|
-
if (size !== maxSize) {
|
|
121780
|
-
dst = sliceArray(dst, 0, size);
|
|
121781
|
-
}
|
|
121782
|
-
return dst;
|
|
121783
|
-
}
|
|
121784
|
-
|
|
121785
|
-
// src/utils/RangeList.ts
|
|
121786
|
-
var RangeList = class {
|
|
121787
|
-
ranges = [];
|
|
121788
|
-
/**
|
|
121789
|
-
* Add a range to the list. If it overlaps with existing ranges,
|
|
121790
|
-
* it will split them and increment reference counts accordingly.
|
|
121791
|
-
*/
|
|
121792
|
-
add(start, end) {
|
|
121793
|
-
if (end <= start) {
|
|
121794
|
-
throw new TypeError("End must be greater than start");
|
|
121795
|
-
}
|
|
121796
|
-
const overlappingRanges = [];
|
|
121797
|
-
for (let i = 0; i < this.ranges.length; i++) {
|
|
121798
|
-
const range2 = this.ranges[i];
|
|
121799
|
-
if (start < range2.end && end > range2.start) {
|
|
121800
|
-
overlappingRanges.push({ index: i, range: range2 });
|
|
121801
|
-
}
|
|
121802
|
-
if (range2.data !== null) {
|
|
121803
|
-
throw new Error("Overlapping range already has data");
|
|
121804
|
-
}
|
|
121805
|
-
}
|
|
121806
|
-
if (overlappingRanges.length === 0) {
|
|
121807
|
-
this.ranges.push({ start, end, refCount: 1, data: null });
|
|
121808
|
-
this.ranges.sort((a, b) => a.start - b.start);
|
|
121809
|
-
return;
|
|
121810
|
-
}
|
|
121811
|
-
const newRanges = [];
|
|
121812
|
-
let currentPos = start;
|
|
121813
|
-
for (let i = 0; i < overlappingRanges.length; i++) {
|
|
121814
|
-
const { range: range2 } = overlappingRanges[i];
|
|
121815
|
-
if (currentPos < range2.start) {
|
|
121816
|
-
newRanges.push({
|
|
121817
|
-
start: currentPos,
|
|
121818
|
-
end: range2.start,
|
|
121819
|
-
refCount: 1,
|
|
121820
|
-
data: null
|
|
121821
|
-
});
|
|
121822
|
-
} else if (range2.start < currentPos) {
|
|
121823
|
-
newRanges.push({
|
|
121824
|
-
start: range2.start,
|
|
121825
|
-
end: currentPos,
|
|
121826
|
-
refCount: range2.refCount,
|
|
121827
|
-
data: null
|
|
121828
|
-
});
|
|
121829
|
-
}
|
|
121830
|
-
newRanges.push({
|
|
121831
|
-
start: Math.max(currentPos, range2.start),
|
|
121832
|
-
end: Math.min(end, range2.end),
|
|
121833
|
-
refCount: range2.refCount + 1,
|
|
121834
|
-
data: null
|
|
121835
|
-
});
|
|
121836
|
-
if (range2.end > end) {
|
|
121837
|
-
newRanges.push({
|
|
121838
|
-
start: end,
|
|
121839
|
-
end: range2.end,
|
|
121840
|
-
refCount: range2.refCount,
|
|
121841
|
-
data: null
|
|
121842
|
-
});
|
|
121843
|
-
}
|
|
121844
|
-
currentPos = Math.max(currentPos, range2.end);
|
|
121845
|
-
}
|
|
121846
|
-
if (currentPos < end) {
|
|
121847
|
-
newRanges.push({
|
|
121848
|
-
start: currentPos,
|
|
121849
|
-
end,
|
|
121850
|
-
refCount: 1,
|
|
121851
|
-
data: null
|
|
121852
|
-
});
|
|
121853
|
-
}
|
|
121854
|
-
const firstIndex = overlappingRanges[0].index;
|
|
121855
|
-
const lastIndex = overlappingRanges[overlappingRanges.length - 1].index;
|
|
121856
|
-
this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges);
|
|
121857
|
-
this.ranges.sort((a, b) => a.start - b.start);
|
|
121858
|
-
}
|
|
121859
|
-
/**
|
|
121860
|
-
* Remove a range from the list. The range must start and end at existing boundaries.
|
|
121861
|
-
*/
|
|
121862
|
-
remove(start, end) {
|
|
121863
|
-
if (end <= start) {
|
|
121864
|
-
throw new TypeError("End must be greater than start");
|
|
121865
|
-
}
|
|
121866
|
-
const affectedRanges = [];
|
|
121867
|
-
for (let i = 0; i < this.ranges.length; i++) {
|
|
121868
|
-
const range2 = this.ranges[i];
|
|
121869
|
-
if (start < range2.end && end > range2.start) {
|
|
121870
|
-
affectedRanges.push({ index: i, range: range2 });
|
|
121871
|
-
}
|
|
121872
|
-
}
|
|
121873
|
-
if (affectedRanges.length === 0) {
|
|
121874
|
-
throw new Error("No ranges found to remove");
|
|
121875
|
-
}
|
|
121876
|
-
if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) {
|
|
121877
|
-
throw new Error("Range boundaries must match existing boundaries");
|
|
121878
|
-
}
|
|
121879
|
-
for (let i = 0; i < affectedRanges.length; i++) {
|
|
121880
|
-
const { range: range2 } = affectedRanges[i];
|
|
121881
|
-
range2.refCount--;
|
|
121882
|
-
}
|
|
121883
|
-
this.ranges = this.ranges.filter((range2) => range2.refCount > 0);
|
|
121884
|
-
}
|
|
121885
|
-
/**
|
|
121886
|
-
* Get all ranges within the specified boundaries.
|
|
121887
|
-
*/
|
|
121888
|
-
getRanges(start, end) {
|
|
121889
|
-
if (end <= start) {
|
|
121890
|
-
throw new TypeError("End must be greater than start");
|
|
121891
|
-
}
|
|
121892
|
-
return this.ranges.filter((range2) => start < range2.end && end > range2.start);
|
|
121893
|
-
}
|
|
121894
|
-
/**
|
|
121895
|
-
* Get all ranges in the list
|
|
121896
|
-
*/
|
|
121897
|
-
getAllRanges() {
|
|
121898
|
-
return [...this.ranges];
|
|
121899
|
-
}
|
|
121900
|
-
};
|
|
121901
|
-
|
|
121902
|
-
// src/utils/XetBlob.ts
|
|
121903
|
-
var JWT_SAFETY_PERIOD = 6e4;
|
|
121904
|
-
var JWT_CACHE_SIZE = 1e3;
|
|
121905
|
-
var compressionSchemeLabels = {
|
|
121906
|
-
[0 /* None */]: "None",
|
|
121907
|
-
[1 /* LZ4 */]: "LZ4",
|
|
121908
|
-
[2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4"
|
|
121909
|
-
};
|
|
121910
|
-
var XET_CHUNK_HEADER_BYTES = 8;
|
|
121911
|
-
var XetBlob = class extends Blob {
|
|
121912
|
-
fetch;
|
|
121913
|
-
accessToken;
|
|
121914
|
-
refreshUrl;
|
|
121915
|
-
reconstructionUrl;
|
|
121916
|
-
hash;
|
|
121917
|
-
start = 0;
|
|
121918
|
-
end = 0;
|
|
121919
|
-
internalLogging = false;
|
|
121920
|
-
reconstructionInfo;
|
|
121921
|
-
listener;
|
|
121922
|
-
constructor(params) {
|
|
121923
|
-
super([]);
|
|
121924
|
-
this.fetch = params.fetch ?? fetch.bind(globalThis);
|
|
121925
|
-
this.accessToken = checkCredentials(params);
|
|
121926
|
-
this.refreshUrl = params.refreshUrl;
|
|
121927
|
-
this.end = params.size;
|
|
121928
|
-
this.reconstructionUrl = params.reconstructionUrl;
|
|
121929
|
-
this.hash = params.hash;
|
|
121930
|
-
this.listener = params.listener;
|
|
121931
|
-
this.internalLogging = params.internalLogging ?? false;
|
|
121932
|
-
this.refreshUrl;
|
|
121933
|
-
}
|
|
121934
|
-
get size() {
|
|
121935
|
-
return this.end - this.start;
|
|
121936
|
-
}
|
|
121937
|
-
#clone() {
|
|
121938
|
-
const blob = new XetBlob({
|
|
121939
|
-
fetch: this.fetch,
|
|
121940
|
-
hash: this.hash,
|
|
121941
|
-
refreshUrl: this.refreshUrl,
|
|
121942
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
121943
|
-
reconstructionUrl: this.reconstructionUrl,
|
|
121944
|
-
size: this.size
|
|
121945
|
-
});
|
|
121946
|
-
blob.accessToken = this.accessToken;
|
|
121947
|
-
blob.start = this.start;
|
|
121948
|
-
blob.end = this.end;
|
|
121949
|
-
blob.reconstructionInfo = this.reconstructionInfo;
|
|
121950
|
-
blob.listener = this.listener;
|
|
121951
|
-
blob.internalLogging = this.internalLogging;
|
|
121952
|
-
return blob;
|
|
121953
|
-
}
|
|
121954
|
-
slice(start = 0, end = this.size) {
|
|
121955
|
-
const slice = this.#clone();
|
|
121956
|
-
slice.start = this.start + start;
|
|
121957
|
-
slice.end = Math.min(this.start + end, this.end);
|
|
121958
|
-
if (slice.start !== this.start || slice.end !== this.end) {
|
|
121959
|
-
slice.reconstructionInfo = void 0;
|
|
122220
|
+
if (this.maxDepth !== Infinity && t.depth() >= this.maxDepth) {
|
|
122221
|
+
continue;
|
|
122222
|
+
}
|
|
122223
|
+
tasks++;
|
|
122224
|
+
const children = t.readdirSync();
|
|
122225
|
+
this.walkCB3Sync(t, children, processor, next);
|
|
122226
|
+
}
|
|
122227
|
+
next();
|
|
121960
122228
|
}
|
|
121961
|
-
|
|
121962
|
-
|
|
121963
|
-
|
|
121964
|
-
|
|
121965
|
-
|
|
121966
|
-
|
|
122229
|
+
walkCB3Sync(target, entries, processor, cb) {
|
|
122230
|
+
processor = processor.filterEntries(target, entries);
|
|
122231
|
+
let tasks = 1;
|
|
122232
|
+
const next = () => {
|
|
122233
|
+
if (--tasks === 0)
|
|
122234
|
+
cb();
|
|
122235
|
+
};
|
|
122236
|
+
for (const [m, absolute, ifDir] of processor.matches.entries()) {
|
|
122237
|
+
if (this.#ignored(m))
|
|
122238
|
+
continue;
|
|
122239
|
+
this.matchSync(m, absolute, ifDir);
|
|
122240
|
+
}
|
|
122241
|
+
for (const [target, patterns] of processor.subwalks.entries()) {
|
|
122242
|
+
tasks++;
|
|
122243
|
+
this.walkCB2Sync(target, patterns, processor.child(), next);
|
|
122244
|
+
}
|
|
122245
|
+
next();
|
|
121967
122246
|
}
|
|
121968
|
-
|
|
121969
|
-
|
|
121970
|
-
|
|
121971
|
-
|
|
121972
|
-
|
|
121973
|
-
|
|
122247
|
+
}
|
|
122248
|
+
class GlobWalker extends GlobUtil {
|
|
122249
|
+
matches = new Set();
|
|
122250
|
+
constructor(patterns, path, opts) {
|
|
122251
|
+
super(patterns, path, opts);
|
|
122252
|
+
}
|
|
122253
|
+
matchEmit(e) {
|
|
122254
|
+
this.matches.add(e);
|
|
122255
|
+
}
|
|
122256
|
+
async walk() {
|
|
122257
|
+
if (this.signal?.aborted)
|
|
122258
|
+
throw this.signal.reason;
|
|
122259
|
+
if (this.path.isUnknown()) {
|
|
122260
|
+
await this.path.lstat();
|
|
121974
122261
|
}
|
|
121975
|
-
|
|
121976
|
-
|
|
121977
|
-
|
|
121978
|
-
|
|
121979
|
-
|
|
121980
|
-
|
|
121981
|
-
|
|
121982
|
-
|
|
121983
|
-
|
|
121984
|
-
|
|
121985
|
-
|
|
121986
|
-
await this.#loadReconstructionInfo();
|
|
122262
|
+
await new Promise((res, rej) => {
|
|
122263
|
+
this.walkCB(this.path, this.patterns, () => {
|
|
122264
|
+
if (this.signal?.aborted) {
|
|
122265
|
+
rej(this.signal.reason);
|
|
122266
|
+
}
|
|
122267
|
+
else {
|
|
122268
|
+
res(this.matches);
|
|
122269
|
+
}
|
|
122270
|
+
});
|
|
122271
|
+
});
|
|
122272
|
+
return this.matches;
|
|
121987
122273
|
}
|
|
121988
|
-
|
|
121989
|
-
|
|
121990
|
-
|
|
122274
|
+
walkSync() {
|
|
122275
|
+
if (this.signal?.aborted)
|
|
122276
|
+
throw this.signal.reason;
|
|
122277
|
+
if (this.path.isUnknown()) {
|
|
122278
|
+
this.path.lstatSync();
|
|
122279
|
+
}
|
|
122280
|
+
// nothing for the callback to do, because this never pauses
|
|
122281
|
+
this.walkCBSync(this.path, this.patterns, () => {
|
|
122282
|
+
if (this.signal?.aborted)
|
|
122283
|
+
throw this.signal.reason;
|
|
122284
|
+
});
|
|
122285
|
+
return this.matches;
|
|
121991
122286
|
}
|
|
121992
|
-
|
|
121993
|
-
|
|
121994
|
-
|
|
121995
|
-
|
|
121996
|
-
|
|
121997
|
-
|
|
121998
|
-
|
|
122287
|
+
}
|
|
122288
|
+
class GlobStream extends GlobUtil {
|
|
122289
|
+
results;
|
|
122290
|
+
constructor(patterns, path, opts) {
|
|
122291
|
+
super(patterns, path, opts);
|
|
122292
|
+
this.results = new Minipass({
|
|
122293
|
+
signal: this.signal,
|
|
122294
|
+
objectMode: true,
|
|
122295
|
+
});
|
|
122296
|
+
this.results.on('drain', () => this.resume());
|
|
122297
|
+
this.results.on('resume', () => this.resume());
|
|
121999
122298
|
}
|
|
122000
|
-
|
|
122001
|
-
|
|
122002
|
-
|
|
122003
|
-
|
|
122004
|
-
|
|
122005
|
-
|
|
122006
|
-
|
|
122007
|
-
if (
|
|
122008
|
-
|
|
122299
|
+
matchEmit(e) {
|
|
122300
|
+
this.results.write(e);
|
|
122301
|
+
if (!this.results.flowing)
|
|
122302
|
+
this.pause();
|
|
122303
|
+
}
|
|
122304
|
+
stream() {
|
|
122305
|
+
const target = this.path;
|
|
122306
|
+
if (target.isUnknown()) {
|
|
122307
|
+
target.lstat().then(() => {
|
|
122308
|
+
this.walkCB(target, this.patterns, () => this.results.end());
|
|
122309
|
+
});
|
|
122009
122310
|
}
|
|
122010
|
-
|
|
122011
|
-
|
|
122012
|
-
throw new Error(`Failed to find range list for term ${term.hash}`);
|
|
122311
|
+
else {
|
|
122312
|
+
this.walkCB(target, this.patterns, () => this.results.end());
|
|
122013
122313
|
}
|
|
122014
|
-
|
|
122015
|
-
|
|
122016
|
-
|
|
122017
|
-
|
|
122018
|
-
|
|
122019
|
-
for (const range2 of termRanges) {
|
|
122020
|
-
for (let chunk2 of range2.data) {
|
|
122021
|
-
if (readBytesToSkip) {
|
|
122022
|
-
const skipped = Math.min(readBytesToSkip, chunk2.byteLength);
|
|
122023
|
-
chunk2 = chunk2.slice(skipped);
|
|
122024
|
-
readBytesToSkip -= skipped;
|
|
122025
|
-
if (!chunk2.byteLength) {
|
|
122026
|
-
continue;
|
|
122027
|
-
}
|
|
122028
|
-
}
|
|
122029
|
-
if (chunk2.byteLength > maxBytes - totalBytesRead) {
|
|
122030
|
-
chunk2 = chunk2.slice(0, maxBytes - totalBytesRead);
|
|
122031
|
-
}
|
|
122032
|
-
totalBytesRead += chunk2.byteLength;
|
|
122033
|
-
yield range2.refCount > 1 ? chunk2.slice() : chunk2;
|
|
122034
|
-
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
122035
|
-
if (totalBytesRead >= maxBytes) {
|
|
122036
|
-
break rangeLoop;
|
|
122037
|
-
}
|
|
122038
|
-
}
|
|
122039
|
-
}
|
|
122040
|
-
rangeList.remove(term.range.start, term.range.end);
|
|
122041
|
-
continue;
|
|
122042
|
-
}
|
|
122314
|
+
return this.results;
|
|
122315
|
+
}
|
|
122316
|
+
streamSync() {
|
|
122317
|
+
if (this.path.isUnknown()) {
|
|
122318
|
+
this.path.lstatSync();
|
|
122043
122319
|
}
|
|
122044
|
-
|
|
122045
|
-
|
|
122046
|
-
|
|
122047
|
-
|
|
122048
|
-
|
|
122049
|
-
|
|
122050
|
-
|
|
122320
|
+
this.walkCBSync(this.path, this.patterns, () => this.results.end());
|
|
122321
|
+
return this.results;
|
|
122322
|
+
}
|
|
122323
|
+
}
|
|
122324
|
+
|
|
122325
|
+
// if no process global, just call it linux.
|
|
122326
|
+
// so we default to case-sensitive, / separators
|
|
122327
|
+
const defaultPlatform = (typeof process === 'object' &&
|
|
122328
|
+
process &&
|
|
122329
|
+
typeof process.platform === 'string') ?
|
|
122330
|
+
process.platform
|
|
122331
|
+
: 'linux';
|
|
122332
|
+
/**
|
|
122333
|
+
* An object that can perform glob pattern traversals.
|
|
122334
|
+
*/
|
|
122335
|
+
class Glob {
|
|
122336
|
+
absolute;
|
|
122337
|
+
cwd;
|
|
122338
|
+
root;
|
|
122339
|
+
dot;
|
|
122340
|
+
dotRelative;
|
|
122341
|
+
follow;
|
|
122342
|
+
ignore;
|
|
122343
|
+
magicalBraces;
|
|
122344
|
+
mark;
|
|
122345
|
+
matchBase;
|
|
122346
|
+
maxDepth;
|
|
122347
|
+
nobrace;
|
|
122348
|
+
nocase;
|
|
122349
|
+
nodir;
|
|
122350
|
+
noext;
|
|
122351
|
+
noglobstar;
|
|
122352
|
+
pattern;
|
|
122353
|
+
platform;
|
|
122354
|
+
realpath;
|
|
122355
|
+
scurry;
|
|
122356
|
+
stat;
|
|
122357
|
+
signal;
|
|
122358
|
+
windowsPathsNoEscape;
|
|
122359
|
+
withFileTypes;
|
|
122360
|
+
includeChildMatches;
|
|
122361
|
+
/**
|
|
122362
|
+
* The options provided to the constructor.
|
|
122363
|
+
*/
|
|
122364
|
+
opts;
|
|
122365
|
+
/**
|
|
122366
|
+
* An array of parsed immutable {@link Pattern} objects.
|
|
122367
|
+
*/
|
|
122368
|
+
patterns;
|
|
122369
|
+
/**
|
|
122370
|
+
* All options are stored as properties on the `Glob` object.
|
|
122371
|
+
*
|
|
122372
|
+
* See {@link GlobOptions} for full options descriptions.
|
|
122373
|
+
*
|
|
122374
|
+
* Note that a previous `Glob` object can be passed as the
|
|
122375
|
+
* `GlobOptions` to another `Glob` instantiation to re-use settings
|
|
122376
|
+
* and caches with a new pattern.
|
|
122377
|
+
*
|
|
122378
|
+
* Traversal functions can be called multiple times to run the walk
|
|
122379
|
+
* again.
|
|
122380
|
+
*/
|
|
122381
|
+
constructor(pattern, opts) {
|
|
122382
|
+
/* c8 ignore start */
|
|
122383
|
+
if (!opts)
|
|
122384
|
+
throw new TypeError('glob options required');
|
|
122385
|
+
/* c8 ignore stop */
|
|
122386
|
+
this.withFileTypes = !!opts.withFileTypes;
|
|
122387
|
+
this.signal = opts.signal;
|
|
122388
|
+
this.follow = !!opts.follow;
|
|
122389
|
+
this.dot = !!opts.dot;
|
|
122390
|
+
this.dotRelative = !!opts.dotRelative;
|
|
122391
|
+
this.nodir = !!opts.nodir;
|
|
122392
|
+
this.mark = !!opts.mark;
|
|
122393
|
+
if (!opts.cwd) {
|
|
122394
|
+
this.cwd = '';
|
|
122051
122395
|
}
|
|
122052
|
-
|
|
122053
|
-
|
|
122054
|
-
log("readBytesToSkip", readBytesToSkip);
|
|
122055
|
-
let resp = await customFetch(fetchInfo.url, {
|
|
122056
|
-
headers: {
|
|
122057
|
-
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
122058
|
-
}
|
|
122059
|
-
});
|
|
122060
|
-
if (resp.status === 403) {
|
|
122061
|
-
reconstructionInfo = await reloadReconstructionInfo();
|
|
122062
|
-
resp = await customFetch(fetchInfo.url, {
|
|
122063
|
-
headers: {
|
|
122064
|
-
Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`
|
|
122065
|
-
}
|
|
122066
|
-
});
|
|
122396
|
+
else if (opts.cwd instanceof URL || opts.cwd.startsWith('file://')) {
|
|
122397
|
+
opts.cwd = require$$0$l.fileURLToPath(opts.cwd);
|
|
122067
122398
|
}
|
|
122068
|
-
|
|
122069
|
-
|
|
122399
|
+
this.cwd = opts.cwd || '';
|
|
122400
|
+
this.root = opts.root;
|
|
122401
|
+
this.magicalBraces = !!opts.magicalBraces;
|
|
122402
|
+
this.nobrace = !!opts.nobrace;
|
|
122403
|
+
this.noext = !!opts.noext;
|
|
122404
|
+
this.realpath = !!opts.realpath;
|
|
122405
|
+
this.absolute = opts.absolute;
|
|
122406
|
+
this.includeChildMatches = opts.includeChildMatches !== false;
|
|
122407
|
+
this.noglobstar = !!opts.noglobstar;
|
|
122408
|
+
this.matchBase = !!opts.matchBase;
|
|
122409
|
+
this.maxDepth =
|
|
122410
|
+
typeof opts.maxDepth === 'number' ? opts.maxDepth : Infinity;
|
|
122411
|
+
this.stat = !!opts.stat;
|
|
122412
|
+
this.ignore = opts.ignore;
|
|
122413
|
+
if (this.withFileTypes && this.absolute !== undefined) {
|
|
122414
|
+
throw new Error('cannot set absolute and withFileTypes:true');
|
|
122070
122415
|
}
|
|
122071
|
-
|
|
122072
|
-
|
|
122073
|
-
resp.headers.get("content-length"),
|
|
122074
|
-
"range",
|
|
122075
|
-
fetchInfo.url_range,
|
|
122076
|
-
resp.headers.get("content-range")
|
|
122077
|
-
);
|
|
122078
|
-
const reader = resp.body?.getReader();
|
|
122079
|
-
if (!reader) {
|
|
122080
|
-
throw new Error("Failed to get reader from response body");
|
|
122416
|
+
if (typeof pattern === 'string') {
|
|
122417
|
+
pattern = [pattern];
|
|
122081
122418
|
}
|
|
122082
|
-
|
|
122083
|
-
|
|
122084
|
-
|
|
122085
|
-
|
|
122086
|
-
|
|
122087
|
-
|
|
122088
|
-
|
|
122089
|
-
|
|
122090
|
-
|
|
122091
|
-
|
|
122092
|
-
log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip);
|
|
122093
|
-
if (!result.value) {
|
|
122094
|
-
log("no data in result, cancelled", result);
|
|
122095
|
-
continue;
|
|
122096
|
-
}
|
|
122097
|
-
totalFetchBytes += result.value.byteLength;
|
|
122098
|
-
if (leftoverBytes) {
|
|
122099
|
-
result.value = combineUint8Arrays(leftoverBytes, result.value);
|
|
122100
|
-
leftoverBytes = void 0;
|
|
122419
|
+
this.windowsPathsNoEscape =
|
|
122420
|
+
!!opts.windowsPathsNoEscape ||
|
|
122421
|
+
opts.allowWindowsEscape ===
|
|
122422
|
+
false;
|
|
122423
|
+
if (this.windowsPathsNoEscape) {
|
|
122424
|
+
pattern = pattern.map(p => p.replace(/\\/g, '/'));
|
|
122425
|
+
}
|
|
122426
|
+
if (this.matchBase) {
|
|
122427
|
+
if (opts.noglobstar) {
|
|
122428
|
+
throw new TypeError('base matching requires globstar');
|
|
122101
122429
|
}
|
|
122102
|
-
|
|
122103
|
-
|
|
122104
|
-
|
|
122105
|
-
|
|
122106
|
-
|
|
122107
|
-
|
|
122108
|
-
|
|
122109
|
-
|
|
122110
|
-
|
|
122111
|
-
|
|
122112
|
-
uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16
|
|
122113
|
-
};
|
|
122114
|
-
log("chunk header", chunkHeader, "to skip", readBytesToSkip);
|
|
122115
|
-
if (chunkHeader.version !== 0) {
|
|
122116
|
-
throw new Error(`Unsupported chunk version ${chunkHeader.version}`);
|
|
122117
|
-
}
|
|
122118
|
-
if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) {
|
|
122119
|
-
throw new Error(
|
|
122120
|
-
`Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}`
|
|
122121
|
-
);
|
|
122122
|
-
}
|
|
122123
|
-
if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) {
|
|
122124
|
-
leftoverBytes = result.value;
|
|
122125
|
-
continue fetchData;
|
|
122126
|
-
}
|
|
122127
|
-
result.value = result.value.slice(XET_CHUNK_HEADER_BYTES);
|
|
122128
|
-
let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes(
|
|
122129
|
-
decompress(
|
|
122130
|
-
result.value.slice(0, chunkHeader.compressed_length),
|
|
122131
|
-
chunkHeader.uncompressed_length
|
|
122132
|
-
)
|
|
122133
|
-
) : result.value.slice(0, chunkHeader.compressed_length);
|
|
122134
|
-
const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end);
|
|
122135
|
-
const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end;
|
|
122136
|
-
const minRefCountToStore = shouldYield ? 2 : 1;
|
|
122137
|
-
let stored = false;
|
|
122138
|
-
if (range2 && range2.refCount >= minRefCountToStore) {
|
|
122139
|
-
range2.data ??= [];
|
|
122140
|
-
range2.data.push(uncompressed);
|
|
122141
|
-
stored = true;
|
|
122142
|
-
}
|
|
122143
|
-
if (shouldYield) {
|
|
122144
|
-
if (readBytesToSkip) {
|
|
122145
|
-
const skipped = Math.min(readBytesToSkip, uncompressed.byteLength);
|
|
122146
|
-
uncompressed = uncompressed.slice(readBytesToSkip);
|
|
122147
|
-
readBytesToSkip -= skipped;
|
|
122148
|
-
}
|
|
122149
|
-
if (uncompressed.byteLength > maxBytes - totalBytesRead) {
|
|
122150
|
-
uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead);
|
|
122151
|
-
}
|
|
122152
|
-
if (uncompressed.byteLength) {
|
|
122153
|
-
log(
|
|
122154
|
-
"yield",
|
|
122155
|
-
uncompressed.byteLength,
|
|
122156
|
-
"bytes",
|
|
122157
|
-
result.value.byteLength,
|
|
122158
|
-
"total read",
|
|
122159
|
-
totalBytesRead,
|
|
122160
|
-
stored
|
|
122161
|
-
);
|
|
122162
|
-
totalBytesRead += uncompressed.byteLength;
|
|
122163
|
-
yield stored ? uncompressed.slice() : uncompressed;
|
|
122164
|
-
listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } });
|
|
122165
|
-
}
|
|
122166
|
-
}
|
|
122167
|
-
chunkIndex++;
|
|
122168
|
-
result.value = result.value.slice(chunkHeader.compressed_length);
|
|
122430
|
+
pattern = pattern.map(p => (p.includes('/') ? p : `./**/${p}`));
|
|
122431
|
+
}
|
|
122432
|
+
this.pattern = pattern;
|
|
122433
|
+
this.platform = opts.platform || defaultPlatform;
|
|
122434
|
+
this.opts = { ...opts, platform: this.platform };
|
|
122435
|
+
if (opts.scurry) {
|
|
122436
|
+
this.scurry = opts.scurry;
|
|
122437
|
+
if (opts.nocase !== undefined &&
|
|
122438
|
+
opts.nocase !== opts.scurry.nocase) {
|
|
122439
|
+
throw new Error('nocase option contradicts provided scurry option');
|
|
122169
122440
|
}
|
|
122170
|
-
}
|
|
122171
|
-
if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) {
|
|
122172
|
-
log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes);
|
|
122173
|
-
log("failed to fetch all data for term", term.hash);
|
|
122174
|
-
throw new Error(
|
|
122175
|
-
`Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}`
|
|
122176
|
-
);
|
|
122177
122441
|
}
|
|
122178
|
-
|
|
122179
|
-
|
|
122180
|
-
|
|
122181
|
-
|
|
122442
|
+
else {
|
|
122443
|
+
const Scurry = opts.platform === 'win32' ? PathScurryWin32
|
|
122444
|
+
: opts.platform === 'darwin' ? PathScurryDarwin
|
|
122445
|
+
: opts.platform ? PathScurryPosix
|
|
122446
|
+
: PathScurry;
|
|
122447
|
+
this.scurry = new Scurry(this.cwd, {
|
|
122448
|
+
nocase: opts.nocase,
|
|
122449
|
+
fs: opts.fs,
|
|
122450
|
+
});
|
|
122451
|
+
}
|
|
122452
|
+
this.nocase = this.scurry.nocase;
|
|
122453
|
+
// If you do nocase:true on a case-sensitive file system, then
|
|
122454
|
+
// we need to use regexps instead of strings for non-magic
|
|
122455
|
+
// path portions, because statting `aBc` won't return results
|
|
122456
|
+
// for the file `AbC` for example.
|
|
122457
|
+
const nocaseMagicOnly = this.platform === 'darwin' || this.platform === 'win32';
|
|
122458
|
+
const mmo = {
|
|
122459
|
+
// default nocase based on platform
|
|
122460
|
+
...opts,
|
|
122461
|
+
dot: this.dot,
|
|
122462
|
+
matchBase: this.matchBase,
|
|
122463
|
+
nobrace: this.nobrace,
|
|
122464
|
+
nocase: this.nocase,
|
|
122465
|
+
nocaseMagicOnly,
|
|
122466
|
+
nocomment: true,
|
|
122467
|
+
noext: this.noext,
|
|
122468
|
+
nonegate: true,
|
|
122469
|
+
optimizationLevel: 2,
|
|
122470
|
+
platform: this.platform,
|
|
122471
|
+
windowsPathsNoEscape: this.windowsPathsNoEscape,
|
|
122472
|
+
debug: !!this.opts.debug,
|
|
122473
|
+
};
|
|
122474
|
+
const mms = this.pattern.map(p => new Minimatch(p, mmo));
|
|
122475
|
+
const [matchSet, globParts] = mms.reduce((set, m) => {
|
|
122476
|
+
set[0].push(...m.set);
|
|
122477
|
+
set[1].push(...m.globParts);
|
|
122478
|
+
return set;
|
|
122479
|
+
}, [[], []]);
|
|
122480
|
+
this.patterns = matchSet.map((set, i) => {
|
|
122481
|
+
const g = globParts[i];
|
|
122482
|
+
/* c8 ignore start */
|
|
122483
|
+
if (!g)
|
|
122484
|
+
throw new Error('invalid pattern object');
|
|
122485
|
+
/* c8 ignore stop */
|
|
122486
|
+
return new Pattern(set, g, 0, this.platform);
|
|
122487
|
+
});
|
|
122182
122488
|
}
|
|
122183
|
-
|
|
122184
|
-
|
|
122185
|
-
|
|
122186
|
-
|
|
122187
|
-
|
|
122188
|
-
|
|
122189
|
-
|
|
122190
|
-
|
|
122191
|
-
|
|
122192
|
-
|
|
122193
|
-
|
|
122194
|
-
|
|
122195
|
-
|
|
122196
|
-
|
|
122197
|
-
|
|
122198
|
-
|
|
122199
|
-
}
|
|
122200
|
-
},
|
|
122201
|
-
type: "bytes"
|
|
122202
|
-
// todo: when Safari supports it, add autoAllocateChunkSize param
|
|
122203
|
-
},
|
|
122204
|
-
// todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function
|
|
122205
|
-
{
|
|
122206
|
-
highWaterMark: 1e3
|
|
122207
|
-
// 1_000 chunks for ~1MB of RAM
|
|
122208
|
-
}
|
|
122209
|
-
);
|
|
122210
|
-
}
|
|
122211
|
-
async arrayBuffer() {
|
|
122212
|
-
const result = await this.#fetch();
|
|
122213
|
-
return new Response(result).arrayBuffer();
|
|
122214
|
-
}
|
|
122215
|
-
async text() {
|
|
122216
|
-
const result = await this.#fetch();
|
|
122217
|
-
return new Response(result).text();
|
|
122218
|
-
}
|
|
122219
|
-
async response() {
|
|
122220
|
-
const result = await this.#fetch();
|
|
122221
|
-
return new Response(result);
|
|
122222
|
-
}
|
|
122223
|
-
stream() {
|
|
122224
|
-
const stream = new TransformStream();
|
|
122225
|
-
this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
122226
|
-
return stream.readable;
|
|
122227
|
-
}
|
|
122228
|
-
};
|
|
122229
|
-
var jwtPromises = /* @__PURE__ */ new Map();
|
|
122230
|
-
var jwts = /* @__PURE__ */ new Map();
|
|
122231
|
-
function cacheKey(params) {
|
|
122232
|
-
return JSON.stringify([params.refreshUrl, params.initialAccessToken]);
|
|
122233
|
-
}
|
|
122234
|
-
function bg4_regroup_bytes(bytes) {
|
|
122235
|
-
const split = Math.floor(bytes.byteLength / 4);
|
|
122236
|
-
const rem = bytes.byteLength % 4;
|
|
122237
|
-
const g1_pos = split + (rem >= 1 ? 1 : 0);
|
|
122238
|
-
const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0);
|
|
122239
|
-
const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0);
|
|
122240
|
-
const ret = new Uint8Array(bytes.byteLength);
|
|
122241
|
-
for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) {
|
|
122242
|
-
ret[i] = bytes[j];
|
|
122243
|
-
}
|
|
122244
|
-
for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) {
|
|
122245
|
-
ret[i] = bytes[j];
|
|
122246
|
-
}
|
|
122247
|
-
for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) {
|
|
122248
|
-
ret[i] = bytes[j];
|
|
122249
|
-
}
|
|
122250
|
-
for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) {
|
|
122251
|
-
ret[i] = bytes[j];
|
|
122252
|
-
}
|
|
122253
|
-
return ret;
|
|
122254
|
-
}
|
|
122255
|
-
async function getAccessToken(initialAccessToken, customFetch, refreshUrl) {
|
|
122256
|
-
const key = cacheKey({ refreshUrl, initialAccessToken });
|
|
122257
|
-
const jwt = jwts.get(key);
|
|
122258
|
-
if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) {
|
|
122259
|
-
return { accessToken: jwt.accessToken, casUrl: jwt.casUrl };
|
|
122260
|
-
}
|
|
122261
|
-
const existingPromise = jwtPromises.get(key);
|
|
122262
|
-
if (existingPromise) {
|
|
122263
|
-
return existingPromise;
|
|
122264
|
-
}
|
|
122265
|
-
const promise = (async () => {
|
|
122266
|
-
const resp = await customFetch(refreshUrl, {
|
|
122267
|
-
headers: {
|
|
122268
|
-
...initialAccessToken ? {
|
|
122269
|
-
Authorization: `Bearer ${initialAccessToken}`
|
|
122270
|
-
} : {}
|
|
122271
|
-
}
|
|
122272
|
-
});
|
|
122273
|
-
if (!resp.ok) {
|
|
122274
|
-
throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`);
|
|
122489
|
+
async walk() {
|
|
122490
|
+
// Walkers always return array of Path objects, so we just have to
|
|
122491
|
+
// coerce them into the right shape. It will have already called
|
|
122492
|
+
// realpath() if the option was set to do so, so we know that's cached.
|
|
122493
|
+
// start out knowing the cwd, at least
|
|
122494
|
+
return [
|
|
122495
|
+
...(await new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
122496
|
+
...this.opts,
|
|
122497
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122498
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122499
|
+
: Infinity,
|
|
122500
|
+
platform: this.platform,
|
|
122501
|
+
nocase: this.nocase,
|
|
122502
|
+
includeChildMatches: this.includeChildMatches,
|
|
122503
|
+
}).walk()),
|
|
122504
|
+
];
|
|
122275
122505
|
}
|
|
122276
|
-
|
|
122277
|
-
|
|
122278
|
-
|
|
122279
|
-
|
|
122280
|
-
|
|
122281
|
-
|
|
122282
|
-
|
|
122283
|
-
|
|
122284
|
-
|
|
122285
|
-
|
|
122286
|
-
|
|
122287
|
-
|
|
122288
|
-
}
|
|
122506
|
+
walkSync() {
|
|
122507
|
+
return [
|
|
122508
|
+
...new GlobWalker(this.patterns, this.scurry.cwd, {
|
|
122509
|
+
...this.opts,
|
|
122510
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122511
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122512
|
+
: Infinity,
|
|
122513
|
+
platform: this.platform,
|
|
122514
|
+
nocase: this.nocase,
|
|
122515
|
+
includeChildMatches: this.includeChildMatches,
|
|
122516
|
+
}).walkSync(),
|
|
122517
|
+
];
|
|
122289
122518
|
}
|
|
122290
|
-
|
|
122291
|
-
|
|
122292
|
-
|
|
122293
|
-
|
|
122294
|
-
|
|
122519
|
+
stream() {
|
|
122520
|
+
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
122521
|
+
...this.opts,
|
|
122522
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122523
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122524
|
+
: Infinity,
|
|
122525
|
+
platform: this.platform,
|
|
122526
|
+
nocase: this.nocase,
|
|
122527
|
+
includeChildMatches: this.includeChildMatches,
|
|
122528
|
+
}).stream();
|
|
122529
|
+
}
|
|
122530
|
+
streamSync() {
|
|
122531
|
+
return new GlobStream(this.patterns, this.scurry.cwd, {
|
|
122532
|
+
...this.opts,
|
|
122533
|
+
maxDepth: this.maxDepth !== Infinity ?
|
|
122534
|
+
this.maxDepth + this.scurry.cwd.depth()
|
|
122535
|
+
: Infinity,
|
|
122536
|
+
platform: this.platform,
|
|
122537
|
+
nocase: this.nocase,
|
|
122538
|
+
includeChildMatches: this.includeChildMatches,
|
|
122539
|
+
}).streamSync();
|
|
122540
|
+
}
|
|
122541
|
+
/**
|
|
122542
|
+
* Default sync iteration function. Returns a Generator that
|
|
122543
|
+
* iterates over the results.
|
|
122544
|
+
*/
|
|
122545
|
+
iterateSync() {
|
|
122546
|
+
return this.streamSync()[Symbol.iterator]();
|
|
122547
|
+
}
|
|
122548
|
+
[Symbol.iterator]() {
|
|
122549
|
+
return this.iterateSync();
|
|
122550
|
+
}
|
|
122551
|
+
/**
|
|
122552
|
+
* Default async iteration function. Returns an AsyncGenerator that
|
|
122553
|
+
* iterates over the results.
|
|
122554
|
+
*/
|
|
122555
|
+
iterate() {
|
|
122556
|
+
return this.stream()[Symbol.asyncIterator]();
|
|
122557
|
+
}
|
|
122558
|
+
[Symbol.asyncIterator]() {
|
|
122559
|
+
return this.iterate();
|
|
122295
122560
|
}
|
|
122296
|
-
jwts.set(key, jwt2);
|
|
122297
|
-
return {
|
|
122298
|
-
accessToken: json.accessToken,
|
|
122299
|
-
casUrl: json.casUrl
|
|
122300
|
-
};
|
|
122301
|
-
})();
|
|
122302
|
-
jwtPromises.set(key, promise);
|
|
122303
|
-
return promise;
|
|
122304
122561
|
}
|
|
122305
122562
|
|
|
122306
|
-
|
|
122307
|
-
|
|
122308
|
-
|
|
122309
|
-
|
|
122310
|
-
|
|
122311
|
-
|
|
122312
|
-
|
|
122313
|
-
|
|
122314
|
-
|
|
122315
|
-
|
|
122316
|
-
|
|
122317
|
-
|
|
122318
|
-
|
|
122319
|
-
|
|
122320
|
-
const supportRange = response.headers.get("accept-ranges") === "bytes";
|
|
122321
|
-
if (!supportRange || size < (opts?.cacheBelow ?? 1e6)) {
|
|
122322
|
-
return await (await customFetch(url)).blob();
|
|
122563
|
+
/**
|
|
122564
|
+
* Return true if the patterns provided contain any magic glob characters,
|
|
122565
|
+
* given the options provided.
|
|
122566
|
+
*
|
|
122567
|
+
* Brace expansion is not considered "magic" unless the `magicalBraces` option
|
|
122568
|
+
* is set, as brace expansion just turns one string into an array of strings.
|
|
122569
|
+
* So a pattern like `'x{a,b}y'` would return `false`, because `'xay'` and
|
|
122570
|
+
* `'xby'` both do not contain any magic glob characters, and it's treated the
|
|
122571
|
+
* same as if you had called it on `['xay', 'xby']`. When `magicalBraces:true`
|
|
122572
|
+
* is in the options, brace expansion _is_ treated as a pattern having magic.
|
|
122573
|
+
*/
|
|
122574
|
+
const hasMagic = (pattern, options = {}) => {
|
|
122575
|
+
if (!Array.isArray(pattern)) {
|
|
122576
|
+
pattern = [pattern];
|
|
122323
122577
|
}
|
|
122324
|
-
|
|
122325
|
-
|
|
122326
|
-
|
|
122327
|
-
start;
|
|
122328
|
-
end;
|
|
122329
|
-
contentType;
|
|
122330
|
-
full;
|
|
122331
|
-
fetch;
|
|
122332
|
-
accessToken;
|
|
122333
|
-
constructor(url, start, end, contentType, full, customFetch, accessToken) {
|
|
122334
|
-
super([]);
|
|
122335
|
-
this.url = url;
|
|
122336
|
-
this.start = start;
|
|
122337
|
-
this.end = end;
|
|
122338
|
-
this.contentType = contentType;
|
|
122339
|
-
this.full = full;
|
|
122340
|
-
this.fetch = customFetch;
|
|
122341
|
-
this.accessToken = accessToken;
|
|
122342
|
-
}
|
|
122343
|
-
get size() {
|
|
122344
|
-
return this.end - this.start;
|
|
122345
|
-
}
|
|
122346
|
-
get type() {
|
|
122347
|
-
return this.contentType;
|
|
122348
|
-
}
|
|
122349
|
-
slice(start = 0, end = this.size) {
|
|
122350
|
-
const slice = new WebBlob(
|
|
122351
|
-
this.url,
|
|
122352
|
-
this.start + start,
|
|
122353
|
-
Math.min(this.start + end, this.end),
|
|
122354
|
-
this.contentType,
|
|
122355
|
-
start === 0 && end === this.size ? this.full : false,
|
|
122356
|
-
this.fetch,
|
|
122357
|
-
this.accessToken
|
|
122358
|
-
);
|
|
122359
|
-
return slice;
|
|
122360
|
-
}
|
|
122361
|
-
async arrayBuffer() {
|
|
122362
|
-
const result = await this.fetchRange();
|
|
122363
|
-
return result.arrayBuffer();
|
|
122364
|
-
}
|
|
122365
|
-
async text() {
|
|
122366
|
-
const result = await this.fetchRange();
|
|
122367
|
-
return result.text();
|
|
122368
|
-
}
|
|
122369
|
-
stream() {
|
|
122370
|
-
const stream = new TransformStream();
|
|
122371
|
-
this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message));
|
|
122372
|
-
return stream.readable;
|
|
122373
|
-
}
|
|
122374
|
-
fetchRange() {
|
|
122375
|
-
const fetch2 = this.fetch;
|
|
122376
|
-
if (this.full) {
|
|
122377
|
-
return fetch2(this.url, {
|
|
122378
|
-
...this.accessToken && {
|
|
122379
|
-
headers: {
|
|
122380
|
-
Authorization: `Bearer ${this.accessToken}`
|
|
122381
|
-
}
|
|
122382
|
-
}
|
|
122383
|
-
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
122578
|
+
for (const p of pattern) {
|
|
122579
|
+
if (new Minimatch(p, options).hasMagic())
|
|
122580
|
+
return true;
|
|
122384
122581
|
}
|
|
122385
|
-
return
|
|
122386
|
-
headers: {
|
|
122387
|
-
Range: `bytes=${this.start}-${this.end - 1}`,
|
|
122388
|
-
...this.accessToken && { Authorization: `Bearer ${this.accessToken}` }
|
|
122389
|
-
}
|
|
122390
|
-
}).then((resp) => resp.ok ? resp : createApiError(resp));
|
|
122391
|
-
}
|
|
122582
|
+
return false;
|
|
122392
122583
|
};
|
|
122393
122584
|
|
|
122394
|
-
|
|
122395
|
-
|
|
122396
|
-
|
|
122397
|
-
|
|
122585
|
+
function globStreamSync(pattern, options = {}) {
|
|
122586
|
+
return new Glob(pattern, options).streamSync();
|
|
122587
|
+
}
|
|
122588
|
+
function globStream(pattern, options = {}) {
|
|
122589
|
+
return new Glob(pattern, options).stream();
|
|
122590
|
+
}
|
|
122591
|
+
function globSync(pattern, options = {}) {
|
|
122592
|
+
return new Glob(pattern, options).walkSync();
|
|
122593
|
+
}
|
|
122594
|
+
async function glob_(pattern, options = {}) {
|
|
122595
|
+
return new Glob(pattern, options).walk();
|
|
122596
|
+
}
|
|
122597
|
+
function globIterateSync(pattern, options = {}) {
|
|
122598
|
+
return new Glob(pattern, options).iterateSync();
|
|
122599
|
+
}
|
|
122600
|
+
function globIterate(pattern, options = {}) {
|
|
122601
|
+
return new Glob(pattern, options).iterate();
|
|
122398
122602
|
}
|
|
122603
|
+
// aliases: glob.sync.stream() glob.stream.sync() glob.sync() etc
|
|
122604
|
+
const streamSync = globStreamSync;
|
|
122605
|
+
const stream = Object.assign(globStream, { sync: globStreamSync });
|
|
122606
|
+
const iterateSync = globIterateSync;
|
|
122607
|
+
const iterate = Object.assign(globIterate, {
|
|
122608
|
+
sync: globIterateSync,
|
|
122609
|
+
});
|
|
122610
|
+
const sync = Object.assign(globSync, {
|
|
122611
|
+
stream: globStreamSync,
|
|
122612
|
+
iterate: globIterateSync,
|
|
122613
|
+
});
|
|
122614
|
+
const glob = Object.assign(glob_, {
|
|
122615
|
+
glob: glob_,
|
|
122616
|
+
globSync,
|
|
122617
|
+
sync,
|
|
122618
|
+
globStream,
|
|
122619
|
+
stream,
|
|
122620
|
+
globStreamSync,
|
|
122621
|
+
streamSync,
|
|
122622
|
+
globIterate,
|
|
122623
|
+
iterate,
|
|
122624
|
+
globIterateSync,
|
|
122625
|
+
iterateSync,
|
|
122626
|
+
Glob,
|
|
122627
|
+
hasMagic,
|
|
122628
|
+
escape: escape$1,
|
|
122629
|
+
unescape: unescape$1,
|
|
122630
|
+
});
|
|
122631
|
+
glob.glob = glob;
|
|
122399
122632
|
|
|
122400
|
-
|
|
122401
|
-
|
|
122402
|
-
|
|
122403
|
-
const repoId = toRepoId(params.repo);
|
|
122404
|
-
const hubUrl = params.hubUrl ?? HUB_URL;
|
|
122405
|
-
const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}/${encodeURIComponent(params.revision ?? "main")}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : "");
|
|
122406
|
-
const resp = await (params.fetch ?? fetch)(url, {
|
|
122407
|
-
method: "GET",
|
|
122408
|
-
headers: {
|
|
122409
|
-
...accessToken && {
|
|
122410
|
-
Authorization: `Bearer ${accessToken}`
|
|
122411
|
-
},
|
|
122412
|
-
Range: "bytes=0-0",
|
|
122413
|
-
Accept: "application/vnd.xet-fileinfo+json, */*"
|
|
122633
|
+
function matchesQuantizationVariant({ filePath, variant }) {
|
|
122634
|
+
if (!variant) {
|
|
122635
|
+
return false;
|
|
122414
122636
|
}
|
|
122415
|
-
|
|
122416
|
-
|
|
122417
|
-
|
|
122418
|
-
|
|
122419
|
-
|
|
122420
|
-
|
|
122421
|
-
}
|
|
122422
|
-
let size;
|
|
122423
|
-
let xetInfo;
|
|
122424
|
-
if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) {
|
|
122425
|
-
size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid");
|
|
122426
|
-
if (isNaN(size)) {
|
|
122427
|
-
throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header");
|
|
122637
|
+
const escapedVariant = variant.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
122638
|
+
const matcher = new RegExp(`(^|[\\-./_])${escapedVariant}(?=[\\-./]|$)`, "i");
|
|
122639
|
+
const normalizedPath = filePath.replace(/\\/g, "/");
|
|
122640
|
+
const segments = normalizedPath.split("/").filter(Boolean);
|
|
122641
|
+
if (segments.length === 0) {
|
|
122642
|
+
return false;
|
|
122428
122643
|
}
|
|
122429
|
-
const
|
|
122430
|
-
|
|
122431
|
-
|
|
122432
|
-
try {
|
|
122433
|
-
return new URL(links["xet-reconstruction-info"]);
|
|
122434
|
-
} catch {
|
|
122435
|
-
return null;
|
|
122436
|
-
}
|
|
122437
|
-
})();
|
|
122438
|
-
const refreshUrl = (() => {
|
|
122439
|
-
try {
|
|
122440
|
-
return new URL(links["xet-auth"]);
|
|
122441
|
-
} catch {
|
|
122442
|
-
return null;
|
|
122443
|
-
}
|
|
122444
|
-
})();
|
|
122445
|
-
if (!hash2) {
|
|
122446
|
-
throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header");
|
|
122644
|
+
const filename = segments[segments.length - 1].replace(/\.gguf$/i, "");
|
|
122645
|
+
if (matcher.test(filename)) {
|
|
122646
|
+
return true;
|
|
122447
122647
|
}
|
|
122448
|
-
|
|
122449
|
-
|
|
122648
|
+
return segments.slice(0, -1).some(segment => matcher.test(segment));
|
|
122649
|
+
}
|
|
122650
|
+
async function findQuantizedModelTarget({ model, path }) {
|
|
122651
|
+
if (model.source.type === "storage") {
|
|
122652
|
+
throw new Error("Model storage not supported yet");
|
|
122450
122653
|
}
|
|
122451
|
-
|
|
122452
|
-
|
|
122453
|
-
refreshUrl,
|
|
122454
|
-
reconstructionUrl
|
|
122455
|
-
};
|
|
122456
|
-
}
|
|
122457
|
-
if (size === void 0 || isNaN(size)) {
|
|
122458
|
-
const contentRangeHeader = resp.headers.get("content-range");
|
|
122459
|
-
if (!contentRangeHeader) {
|
|
122460
|
-
throw new InvalidApiResponseFormatError("Expected size information");
|
|
122654
|
+
if (model.format !== "gguf") {
|
|
122655
|
+
throw new Error(`Model format not supported: ${model.format}`);
|
|
122461
122656
|
}
|
|
122462
|
-
const [,
|
|
122463
|
-
|
|
122464
|
-
|
|
122465
|
-
|
|
122657
|
+
const [, variant = null] = model.source.slug.split(":");
|
|
122658
|
+
const modelFiles = (await glob("**/*.gguf", {
|
|
122659
|
+
absolute: true,
|
|
122660
|
+
cwd: path,
|
|
122661
|
+
nodir: true
|
|
122662
|
+
})).filter(file => !/(mmproj|clip)/i.test(file));
|
|
122663
|
+
if (modelFiles.length <= 0) {
|
|
122664
|
+
throw new Error(`No models found for format: ${model.format}`);
|
|
122466
122665
|
}
|
|
122467
|
-
|
|
122468
|
-
|
|
122469
|
-
|
|
122470
|
-
|
|
122471
|
-
|
|
122472
|
-
|
|
122473
|
-
|
|
122474
|
-
|
|
122475
|
-
|
|
122476
|
-
// Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it.
|
|
122477
|
-
url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url
|
|
122478
|
-
};
|
|
122479
|
-
}
|
|
122480
|
-
|
|
122481
|
-
// src/lib/download-file.ts
|
|
122482
|
-
async function downloadFile(params) {
|
|
122483
|
-
const accessToken = checkCredentials(params);
|
|
122484
|
-
const info = params.downloadInfo ?? await fileDownloadInfo({
|
|
122485
|
-
accessToken,
|
|
122486
|
-
repo: params.repo,
|
|
122487
|
-
path: params.path,
|
|
122488
|
-
revision: params.revision,
|
|
122489
|
-
hubUrl: params.hubUrl,
|
|
122490
|
-
fetch: params.fetch,
|
|
122491
|
-
raw: params.raw
|
|
122492
|
-
});
|
|
122493
|
-
if (!info) {
|
|
122494
|
-
return null;
|
|
122495
|
-
}
|
|
122496
|
-
if (info.xet && params.xet) {
|
|
122497
|
-
return new XetBlob({
|
|
122498
|
-
refreshUrl: info.xet.refreshUrl.href,
|
|
122499
|
-
reconstructionUrl: info.xet.reconstructionUrl.href,
|
|
122500
|
-
fetch: params.fetch,
|
|
122501
|
-
accessToken,
|
|
122502
|
-
size: info.size
|
|
122503
|
-
});
|
|
122504
|
-
}
|
|
122505
|
-
return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken);
|
|
122666
|
+
modelFiles.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: "base" }));
|
|
122667
|
+
if (!variant) {
|
|
122668
|
+
return modelFiles[0];
|
|
122669
|
+
}
|
|
122670
|
+
const matches = modelFiles.filter(fileName => matchesQuantizationVariant({ filePath: fileName, variant: variant ?? "" }));
|
|
122671
|
+
if (matches.length === 0) {
|
|
122672
|
+
throw new Error(`No model found for format and variant: ${model.format} / ${variant}`);
|
|
122673
|
+
}
|
|
122674
|
+
return matches[0];
|
|
122506
122675
|
}
|
|
122507
122676
|
|
|
122508
|
-
|
|
122509
|
-
|
|
122510
|
-
|
|
122511
|
-
|
|
122512
|
-
|
|
122513
|
-
|
|
122514
|
-
|
|
122515
|
-
|
|
122516
|
-
accept: "application/json",
|
|
122517
|
-
...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0
|
|
122518
|
-
}
|
|
122519
|
-
});
|
|
122520
|
-
if (!res.ok) {
|
|
122521
|
-
throw await createApiError(res);
|
|
122677
|
+
const VLLM_START_ARGS = ["-m", "vllm.entrypoints.openai.api_server", "--host", "0.0.0.0"];
|
|
122678
|
+
const VLLM_EXECUTABLE = "python3";
|
|
122679
|
+
const DEFAULT_VLLM_CONTEXT_LENGTH = 2048;
|
|
122680
|
+
async function startVLLM({ enginePort, targetDirectory }) {
|
|
122681
|
+
const contextLength = Math.max(1, this.contextLength ?? DEFAULT_VLLM_CONTEXT_LENGTH);
|
|
122682
|
+
let modelPath = targetDirectory;
|
|
122683
|
+
if (this.model.format === "gguf") {
|
|
122684
|
+
modelPath = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
|
|
122522
122685
|
}
|
|
122523
|
-
const
|
|
122524
|
-
|
|
122525
|
-
|
|
122686
|
+
const engineConfig = this.engineConfig;
|
|
122687
|
+
const device = typeof engineConfig?.device === "string" ? engineConfig.device : process.env.VLLM_DEVICE;
|
|
122688
|
+
const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : process.env.VLLM_DTYPE;
|
|
122689
|
+
const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
|
|
122690
|
+
const args = [
|
|
122691
|
+
...VLLM_START_ARGS,
|
|
122692
|
+
"--port",
|
|
122693
|
+
String(enginePort),
|
|
122694
|
+
"--model",
|
|
122695
|
+
modelPath,
|
|
122696
|
+
"--served-model-name",
|
|
122697
|
+
SERVED_MODEL_NAME,
|
|
122698
|
+
"--max-model-len",
|
|
122699
|
+
String(contextLength),
|
|
122700
|
+
"--tensor-parallel-size",
|
|
122701
|
+
String(tensorParallelSize)
|
|
122702
|
+
];
|
|
122703
|
+
if (this.model.taskType === "embeddings") {
|
|
122704
|
+
args.push("--task", "embed");
|
|
122526
122705
|
}
|
|
122527
|
-
|
|
122528
|
-
|
|
122529
|
-
|
|
122706
|
+
args.push(...(await getChatTemplateEngineArgs({
|
|
122707
|
+
engine: this.engine,
|
|
122708
|
+
model: this.model,
|
|
122709
|
+
targetDirectory
|
|
122710
|
+
})));
|
|
122711
|
+
if (device) {
|
|
122712
|
+
args.push("--device", device);
|
|
122713
|
+
}
|
|
122714
|
+
if (dtype) {
|
|
122715
|
+
args.push("--dtype", dtype);
|
|
122716
|
+
}
|
|
122717
|
+
args.push(...parseExtraArgs(engineConfig?.extraArgs));
|
|
122718
|
+
if (this.model.multimodalEnabled) {
|
|
122719
|
+
args.push("--limit-mm-per-prompt", process.env.VLLM_MM_LIMIT ?? '{"image":5}');
|
|
122720
|
+
}
|
|
122721
|
+
if (process.env.VLLM_TRUST_REMOTE_CODE === "true") {
|
|
122722
|
+
args.push("--trust-remote-code");
|
|
122723
|
+
}
|
|
122724
|
+
return createEngineProcess({ args, bin: VLLM_EXECUTABLE, logger: this.logger });
|
|
122530
122725
|
}
|
|
122531
122726
|
|
|
122532
122727
|
const ModelDownloadProgressSchema = object$1({
|
|
@@ -123129,6 +123324,11 @@ async function startLlamacpp({ enginePort, targetDirectory }) {
|
|
|
123129
123324
|
"--ctx-size",
|
|
123130
123325
|
String(contextLength)
|
|
123131
123326
|
];
|
|
123327
|
+
args.push(...(await getChatTemplateEngineArgs({
|
|
123328
|
+
engine: this.engine,
|
|
123329
|
+
model: this.model,
|
|
123330
|
+
targetDirectory
|
|
123331
|
+
})));
|
|
123132
123332
|
if (this.model.taskType === "embeddings") {
|
|
123133
123333
|
args.push("--embedding");
|
|
123134
123334
|
}
|
|
@@ -123268,6 +123468,11 @@ async function startSGLang({ enginePort, targetDirectory }) {
|
|
|
123268
123468
|
if (this.model.taskType === "embeddings") {
|
|
123269
123469
|
args.push("--task", "embed");
|
|
123270
123470
|
}
|
|
123471
|
+
args.push(...(await getChatTemplateEngineArgs({
|
|
123472
|
+
engine: this.engine,
|
|
123473
|
+
model: this.model,
|
|
123474
|
+
targetDirectory
|
|
123475
|
+
})));
|
|
123271
123476
|
if (device) {
|
|
123272
123477
|
args.push("--device", device);
|
|
123273
123478
|
}
|
|
@@ -123348,6 +123553,15 @@ class ModelManager extends EventEmitter {
|
|
|
123348
123553
|
this.uniqueName = createModelStorageKey(this.model);
|
|
123349
123554
|
this.modelsDirectory = path$1.join(root, "models");
|
|
123350
123555
|
}
|
|
123556
|
+
async verifyChatTemplate() {
|
|
123557
|
+
return verifyEngineChatTemplate({
|
|
123558
|
+
engine: this.engine,
|
|
123559
|
+
enginePort: this.enginePort,
|
|
123560
|
+
logger: this.logger,
|
|
123561
|
+
model: this.model,
|
|
123562
|
+
targetDirectory: path$1.join(this.modelsDirectory, this.uniqueName)
|
|
123563
|
+
});
|
|
123564
|
+
}
|
|
123351
123565
|
async fetchOpenAI(path, opts) {
|
|
123352
123566
|
switch (this.engine) {
|
|
123353
123567
|
case "exllamav3":
|
|
@@ -123425,6 +123639,12 @@ class ModelManager extends EventEmitter {
|
|
|
123425
123639
|
finally {
|
|
123426
123640
|
await this.releaseDownloadLock();
|
|
123427
123641
|
}
|
|
123642
|
+
await materializeChatTemplate({
|
|
123643
|
+
engine: this.engine,
|
|
123644
|
+
huggingFaceToken: this.model.source.modelSecret,
|
|
123645
|
+
model: this.model,
|
|
123646
|
+
targetDirectory: path$1.join(this.modelsDirectory, this.uniqueName)
|
|
123647
|
+
});
|
|
123428
123648
|
break;
|
|
123429
123649
|
default: {
|
|
123430
123650
|
const engineType = this.engine;
|
|
@@ -124408,7 +124628,50 @@ function stripImagesFromBody(body) {
|
|
|
124408
124628
|
function isPlainObject$3(value) {
|
|
124409
124629
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
124410
124630
|
}
|
|
124411
|
-
|
|
124631
|
+
/**
|
|
124632
|
+
* Builds `chat_template_kwargs` for engines that read template variables at
|
|
124633
|
+
* render time. Only active when the model carries a template override or
|
|
124634
|
+
* thinking config; otherwise the body forwards untouched (engines like vLLM
|
|
124635
|
+
* consume top-level `reasoning_effort` natively for supported models).
|
|
124636
|
+
*
|
|
124637
|
+
* Precedence (highest wins):
|
|
124638
|
+
* 1. Request `chat_template_kwargs` (per key)
|
|
124639
|
+
* 2. Request top-level `reasoning_effort`
|
|
124640
|
+
* 3. Model thinking config defaults
|
|
124641
|
+
*/
|
|
124642
|
+
function applyChatTemplateKwargs({ body, model }) {
|
|
124643
|
+
const hasTemplateOverride = Boolean(model.chatTemplate);
|
|
124644
|
+
const thinkingConfig = model.thinkingConfig ?? null;
|
|
124645
|
+
if (!hasTemplateOverride && !thinkingConfig)
|
|
124646
|
+
return body;
|
|
124647
|
+
const requestKwargs = isPlainObject$3(body.chat_template_kwargs)
|
|
124648
|
+
? body.chat_template_kwargs
|
|
124649
|
+
: null;
|
|
124650
|
+
const requestEffort = typeof body.reasoning_effort === "string" ? body.reasoning_effort : null;
|
|
124651
|
+
const mergedKwargs = {};
|
|
124652
|
+
if (thinkingConfig?.enabled === false) {
|
|
124653
|
+
mergedKwargs.enable_thinking = false;
|
|
124654
|
+
}
|
|
124655
|
+
if (thinkingConfig?.effort) {
|
|
124656
|
+
mergedKwargs.reasoning_effort = thinkingConfig.effort;
|
|
124657
|
+
}
|
|
124658
|
+
if (requestKwargs) {
|
|
124659
|
+
Object.assign(mergedKwargs, requestKwargs);
|
|
124660
|
+
}
|
|
124661
|
+
const hasRequestReasoningEffort = requestKwargs !== null && "reasoning_effort" in requestKwargs;
|
|
124662
|
+
if (requestEffort && !hasRequestReasoningEffort) {
|
|
124663
|
+
mergedKwargs.reasoning_effort = requestEffort;
|
|
124664
|
+
}
|
|
124665
|
+
const payload = { ...body };
|
|
124666
|
+
if (Object.keys(mergedKwargs).length > 0) {
|
|
124667
|
+
payload.chat_template_kwargs = mergedKwargs;
|
|
124668
|
+
}
|
|
124669
|
+
if (requestEffort !== null) {
|
|
124670
|
+
delete payload.reasoning_effort;
|
|
124671
|
+
}
|
|
124672
|
+
return payload;
|
|
124673
|
+
}
|
|
124674
|
+
function serializeRequestBody$1(body, { model, path } = {}) {
|
|
124412
124675
|
if (!isPlainObject$3(body)) {
|
|
124413
124676
|
const payload = typeof body === "string" ? body : JSON.stringify(body);
|
|
124414
124677
|
return {
|
|
@@ -124416,7 +124679,10 @@ function serializeRequestBody$1(body) {
|
|
|
124416
124679
|
payload
|
|
124417
124680
|
};
|
|
124418
124681
|
}
|
|
124419
|
-
|
|
124682
|
+
let requestPayload = { ...body };
|
|
124683
|
+
if (path === "/v1/chat/completions" && model) {
|
|
124684
|
+
requestPayload = applyChatTemplateKwargs({ body: requestPayload, model });
|
|
124685
|
+
}
|
|
124420
124686
|
if (requestPayload.stream === true) {
|
|
124421
124687
|
const streamOptions = requestPayload.stream_options;
|
|
124422
124688
|
const normalizedStreamOptions = isPlainObject$3(streamOptions)
|
|
@@ -124615,7 +124881,7 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
|
|
|
124615
124881
|
const engineType = conduitConfiguration.engineConfig?.type ?? null;
|
|
124616
124882
|
const engineConfig = conduitConfiguration.engineConfig?.config ?? null;
|
|
124617
124883
|
const effectiveBody = modelManager.model.multimodalEnabled ? body : stripImagesFromBody(body);
|
|
124618
|
-
const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(effectiveBody);
|
|
124884
|
+
const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(effectiveBody, { model: modelManager.model, path });
|
|
124619
124885
|
const requestStartedAt = Date.now();
|
|
124620
124886
|
const requestBody = JSON.parse(serializedBody);
|
|
124621
124887
|
const streamRequested = requestBody.stream === true;
|
|
@@ -125001,6 +125267,17 @@ function translateAnthropicRequestToOpenAI(body) {
|
|
|
125001
125267
|
result.top_p = parsed.top_p;
|
|
125002
125268
|
if (Array.isArray(parsed.stop_sequences))
|
|
125003
125269
|
result.stop = parsed.stop_sequences;
|
|
125270
|
+
if (parsed.thinking && typeof parsed.thinking === "object") {
|
|
125271
|
+
const thinking = parsed.thinking;
|
|
125272
|
+
if (thinking.type === "enabled") {
|
|
125273
|
+
const budgetTokens = typeof thinking.budget_tokens === "number" ? thinking.budget_tokens : null;
|
|
125274
|
+
result.chat_template_kwargs = {
|
|
125275
|
+
reasoning_effort: budgetTokens
|
|
125276
|
+
? reasoningEffortForAnthropicBudget(budgetTokens)
|
|
125277
|
+
: "high"
|
|
125278
|
+
};
|
|
125279
|
+
}
|
|
125280
|
+
}
|
|
125004
125281
|
if (Array.isArray(parsed.tools)) {
|
|
125005
125282
|
result.tools = parsed.tools.map((tool) => {
|
|
125006
125283
|
const t = tool;
|
|
@@ -125272,11 +125549,13 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
|
|
|
125272
125549
|
const requestStartedAt = Date.now();
|
|
125273
125550
|
const requestBody = JSON.parse(serializedBody);
|
|
125274
125551
|
const streamRequested = requestBody.stream === true;
|
|
125275
|
-
const
|
|
125276
|
-
|
|
125277
|
-
|
|
125278
|
-
|
|
125279
|
-
|
|
125552
|
+
const translated = needsTranslation ? translateAnthropicRequestToOpenAI(serializedBody) : null;
|
|
125553
|
+
const targetPath = translated?.path ?? "/v1/messages";
|
|
125554
|
+
const targetBody = translated
|
|
125555
|
+
? JSON.stringify(applyChatTemplateKwargs({
|
|
125556
|
+
body: JSON.parse(translated.body),
|
|
125557
|
+
model: modelManager.model
|
|
125558
|
+
}))
|
|
125280
125559
|
: serializedBody;
|
|
125281
125560
|
const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
|
|
125282
125561
|
const promptTokens = normalizeTokenCount(usage?.inputTokens);
|
|
@@ -156174,13 +156453,14 @@ async function createApplication({ abortController, apiClient, configuration, lo
|
|
|
156174
156453
|
});
|
|
156175
156454
|
conduitStateReportManager.reportStateChange();
|
|
156176
156455
|
};
|
|
156177
|
-
const setOnlineState = () => {
|
|
156178
|
-
if (conduitStateManager.getState().state === "online") {
|
|
156456
|
+
const setOnlineState = ({ warnings } = {}) => {
|
|
156457
|
+
if (conduitStateManager.getState().state === "online" && warnings === undefined) {
|
|
156179
156458
|
return;
|
|
156180
156459
|
}
|
|
156181
156460
|
conduitStateManager.setState({
|
|
156182
156461
|
modelName,
|
|
156183
|
-
state: "online"
|
|
156462
|
+
state: "online",
|
|
156463
|
+
...(warnings !== undefined ? { warnings } : {})
|
|
156184
156464
|
});
|
|
156185
156465
|
conduitStateReportManager.reportStateChange();
|
|
156186
156466
|
};
|
|
@@ -156202,6 +156482,23 @@ async function createApplication({ abortController, apiClient, configuration, lo
|
|
|
156202
156482
|
});
|
|
156203
156483
|
modelManager.on("engineReady", () => {
|
|
156204
156484
|
setOnlineState();
|
|
156485
|
+
const readyModelManager = modelManager;
|
|
156486
|
+
readyModelManager
|
|
156487
|
+
.verifyChatTemplate()
|
|
156488
|
+
.then(warning => {
|
|
156489
|
+
if (modelManager !== readyModelManager) {
|
|
156490
|
+
return;
|
|
156491
|
+
}
|
|
156492
|
+
if (warning) {
|
|
156493
|
+
logger.warn("Chat template verification", { warning });
|
|
156494
|
+
setOnlineState({ warnings: [warning] });
|
|
156495
|
+
}
|
|
156496
|
+
})
|
|
156497
|
+
.catch(error => {
|
|
156498
|
+
logger.warn("Chat template verification failed", {
|
|
156499
|
+
error: asError(error)
|
|
156500
|
+
});
|
|
156501
|
+
});
|
|
156205
156502
|
});
|
|
156206
156503
|
modelManager.on("engineTerminated", () => {
|
|
156207
156504
|
if (stopRequestedByControl) {
|