@cmflow/source-pack 3.20.0 → 3.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -15
- package/dist/bin/source-pack.mjs +8 -8
- package/dist/bin/source-pack.mjs.map +1 -1
- package/dist/buildSourcePack-59jTztr3.mjs +3 -0
- package/dist/{buildSourcePack-CaaMZ_tw.mjs → buildSourcePack-YCfD24d4.mjs} +8 -27
- package/dist/{buildSourcePack-CaaMZ_tw.mjs.map → buildSourcePack-YCfD24d4.mjs.map} +1 -1
- package/dist/{constants-DkiLCuIK.mjs → constants-o4tbCIrY.mjs} +3 -4
- package/dist/constants-o4tbCIrY.mjs.map +1 -0
- package/dist/index.d.mts +7 -24
- package/dist/index.mjs +6 -7
- package/dist/index.mjs.map +1 -1
- package/dist/{initConfig-ByGsbShR.mjs → initConfig-Cy7oMMxo.mjs} +5 -6
- package/dist/{initConfig-ByGsbShR.mjs.map → initConfig-Cy7oMMxo.mjs.map} +1 -1
- package/dist/{loadConfig-CNrL90dq.mjs → loadConfig-KYduEj2v.mjs} +3 -5
- package/dist/loadConfig-KYduEj2v.mjs.map +1 -0
- package/dist/{packageVersion-B0749hF6.mjs → packageVersion-CpvQqEy_.mjs} +2 -2
- package/dist/{packageVersion-B0749hF6.mjs.map → packageVersion-CpvQqEy_.mjs.map} +1 -1
- package/dist/{publishSourcePack-xNr9RBgv.mjs → publishSourcePack-BfPLD37B.mjs} +2 -2
- package/dist/{publishSourcePack-xNr9RBgv.mjs.map → publishSourcePack-BfPLD37B.mjs.map} +1 -1
- package/dist/publishSourcePack-BwAIlydQ.mjs +3 -0
- package/package.json +1 -1
- package/dist/buildSourcePack-DIp8G_uL.mjs +0 -3
- package/dist/constants-DkiLCuIK.mjs.map +0 -1
- package/dist/loadConfig-CNrL90dq.mjs.map +0 -1
- package/dist/publishSourcePack-Dj1cdWfT.mjs +0 -3
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as __toESM, i as __commonJSMin, n as require_auth_errors, r as require_token_error, t as require_token_util } from "./token-util-C3JYrW4-.mjs";
|
|
2
|
-
import { C as toJSONSchema, S as safeParseAsync, a as any, b as unknown, c as custom, d as literal, f as looseObject, g as record, h as object, i as _null, l as discriminatedUnion, m as number, n as _enum, o as array, p as never, r as _instanceof, s as boolean, t as packageVersion, u as lazy, v as string, w as errorMessage, y as union } from "./packageVersion-
|
|
3
|
-
import { _ as
|
|
2
|
+
import { C as toJSONSchema, S as safeParseAsync, a as any, b as unknown, c as custom, d as literal, f as looseObject, g as record, h as object, i as _null, l as discriminatedUnion, m as number, n as _enum, o as array, p as never, r as _instanceof, s as boolean, t as packageVersion, u as lazy, v as string, w as errorMessage, y as union } from "./packageVersion-CpvQqEy_.mjs";
|
|
3
|
+
import { _ as SOURCE_PACK_FORMAT_VERSION, d as EMBEDDINGS_FILE, f as EMBEDDING_INPUT_FORMAT, g as PACKAGE_NAME, h as MANIFEST_FILE, n as CHUNKS_FILE, r as CHUNK_ID_ALGORITHM, t as CHUNKING_FORMAT_VERSION } from "./constants-o4tbCIrY.mjs";
|
|
4
4
|
import { s as DEFAULT_MAX_INPUT_CHARS } from "./defaults-BJs7a37e.mjs";
|
|
5
5
|
import { n as silentLogger } from "./logger-D1XGzzjT.mjs";
|
|
6
6
|
import { n as Qn, t as projectIdentity } from "./projectIdentity-F97vs1H-.mjs";
|
|
@@ -11095,8 +11095,8 @@ function encodeChunks(chunks) {
|
|
|
11095
11095
|
})).join("\n");
|
|
11096
11096
|
return gzipAsync(Buffer.from(jsonl ? `${jsonl}\n` : ""), { level: 9 });
|
|
11097
11097
|
}
|
|
11098
|
-
/** Writes `manifest.json`, `chunks.jsonl.gz
|
|
11099
|
-
async function writeSourcePack({ archivePath, manifest, chunks, embeddings
|
|
11098
|
+
/** Writes `manifest.json`, `chunks.jsonl.gz` and `embeddings.f32` into a deterministic `.tar.gz`. */
|
|
11099
|
+
async function writeSourcePack({ archivePath, manifest, chunks, embeddings }) {
|
|
11100
11100
|
const staging = await mkdtemp(path.join(tmpdir(), "source-pack-"));
|
|
11101
11101
|
try {
|
|
11102
11102
|
const entries = [[MANIFEST_FILE, Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`)]];
|
|
@@ -11104,7 +11104,6 @@ async function writeSourcePack({ archivePath, manifest, chunks, embeddings, atta
|
|
|
11104
11104
|
const embeddingsBuffer = embeddings ? encodeEmbeddings(embeddings) : void 0;
|
|
11105
11105
|
entries.push([CHUNKS_FILE, chunksBuffer]);
|
|
11106
11106
|
if (embeddingsBuffer) entries.push([EMBEDDINGS_FILE, embeddingsBuffer]);
|
|
11107
|
-
entries.push(...attachments);
|
|
11108
11107
|
for (const [file, content] of entries) {
|
|
11109
11108
|
const target = path.join(staging, ...file.split("/"));
|
|
11110
11109
|
await mkdir(path.dirname(target), { recursive: true });
|
|
@@ -11145,7 +11144,7 @@ async function resolveVersion(repository, sha, options) {
|
|
|
11145
11144
|
if (typeof version !== "string" || !version) throw new Error("Unable to determine the release version: pass --tag <tag> or --release-version <version>");
|
|
11146
11145
|
return version;
|
|
11147
11146
|
}
|
|
11148
|
-
/** Builds the source pack of `config.repository` at a commit: chunks + embeddings +
|
|
11147
|
+
/** Builds the source pack of `config.repository` at a commit: chunks + embeddings + manifest. */
|
|
11149
11148
|
async function buildSourcePack(options) {
|
|
11150
11149
|
const { config, env = process.env, logger = silentLogger, now = () => /* @__PURE__ */ new Date() } = options;
|
|
11151
11150
|
const dryRun = Boolean(options.dryRun);
|
|
@@ -11175,21 +11174,6 @@ async function buildSourcePack(options) {
|
|
|
11175
11174
|
}
|
|
11176
11175
|
for (const entry of tooLarge) logger.warn(`skipped ${entry.path} (${entry.size} bytes > maxFileBytes ${config.sources.maxFileBytes})`);
|
|
11177
11176
|
logger.info(`${indexed} files -> ${chunks.length} chunks (skipped: ${skippedBinary} binary, ${tooLarge.length} too large)`);
|
|
11178
|
-
const attachments = /* @__PURE__ */ new Map();
|
|
11179
|
-
const attachmentInfos = [];
|
|
11180
|
-
for (const attachmentPath of config.attachments) {
|
|
11181
|
-
const normalized = attachmentPath.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
11182
|
-
const content = await repository.readFile(sha, normalized);
|
|
11183
|
-
if (!content) throw new Error(`Attachment ${normalized} is not tracked by Git at ${sha}`);
|
|
11184
|
-
const file = `${ATTACHMENTS_DIR}/${normalized}`;
|
|
11185
|
-
attachments.set(file, content);
|
|
11186
|
-
attachmentInfos.push({
|
|
11187
|
-
path: normalized,
|
|
11188
|
-
file,
|
|
11189
|
-
bytes: content.length,
|
|
11190
|
-
sha256: sha256(content)
|
|
11191
|
-
});
|
|
11192
|
-
}
|
|
11193
11177
|
const embeddingConfig = options.skipEmbeddings ? null : config.embedding;
|
|
11194
11178
|
let embeddings = null;
|
|
11195
11179
|
if (embeddingConfig && !dryRun) {
|
|
@@ -11241,18 +11225,16 @@ async function buildSourcePack(options) {
|
|
|
11241
11225
|
count: chunks.length,
|
|
11242
11226
|
file: CHUNKS_FILE
|
|
11243
11227
|
},
|
|
11244
|
-
attachments: attachmentInfos,
|
|
11245
11228
|
publish: { assetName }
|
|
11246
11229
|
};
|
|
11247
11230
|
if (dryRun) {
|
|
11248
11231
|
const chunksBytes = (await encodeChunks(chunks)).length;
|
|
11249
11232
|
const embeddingsBytes = embeddingConfig ? chunks.length * embeddingConfig.dimensions * 4 : 0;
|
|
11250
|
-
const attachmentsBytes = attachmentInfos.reduce((total, attachment) => total + attachment.bytes, 0);
|
|
11251
11233
|
return {
|
|
11252
11234
|
manifest,
|
|
11253
11235
|
chunks,
|
|
11254
11236
|
dryRun,
|
|
11255
|
-
bytes: chunksBytes + embeddingsBytes
|
|
11237
|
+
bytes: chunksBytes + embeddingsBytes,
|
|
11256
11238
|
chunksBytes,
|
|
11257
11239
|
embeddingsBytes
|
|
11258
11240
|
};
|
|
@@ -11262,8 +11244,7 @@ async function buildSourcePack(options) {
|
|
|
11262
11244
|
archivePath,
|
|
11263
11245
|
manifest,
|
|
11264
11246
|
chunks,
|
|
11265
|
-
embeddings
|
|
11266
|
-
attachments
|
|
11247
|
+
embeddings
|
|
11267
11248
|
});
|
|
11268
11249
|
await mkdir(config.outputDir, { recursive: true });
|
|
11269
11250
|
await writeFile(path.join(config.outputDir, MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
@@ -11277,4 +11258,4 @@ async function buildSourcePack(options) {
|
|
|
11277
11258
|
|
|
11278
11259
|
//#endregion
|
|
11279
11260
|
export { configFingerprint as a, embeddingInput as i, writeSourcePack as n, chunkText as o, embedTexts as r, buildSourcePack as t };
|
|
11280
|
-
//# sourceMappingURL=buildSourcePack-
|
|
11261
|
+
//# sourceMappingURL=buildSourcePack-YCfD24d4.mjs.map
|