mlclaw 0.1.0

package/src/vendor/hfjs-xet/types/public.ts

@@ -0,0 +1,207 @@
+// @ts-nocheck -- vendored upstream code, kept verbatim; our strict tsconfig does not apply.
+// Vendored from huggingface/huggingface.js@f8fdf6be (packages/hub/src/types/public.ts), MIT License.
+// Delete this directory when bucket support is upstreamed to @huggingface/hub.
+import type { PipelineType } from "@huggingface/tasks";
+
+export type RepoType = "space" | "dataset" | "model" | "bucket" | "kernel";
+
+export interface RepoId {
+	name: string;
+	type: RepoType;
+}
+
+export type RepoFullName =
+	| string
+	| `spaces/${string}`
+	| `datasets/${string}`
+	| `buckets/${string}`
+	| `kernels/${string}`;
+
+export type RepoDesignation = RepoId | RepoFullName;
+
+/**
+ * A {@link RepoDesignation} narrowed to bucket repos.
+ *
+ * Used by APIs that only operate on buckets (e.g. {@link copyFile}, {@link copyFiles},
+ * {@link copyFolder}).
+ */
+export type BucketDesignation = { type: "bucket"; name: string } | `buckets/${string}`;
+
+/** Actually `hf_${string}`, but for convenience, using the string type */
+export type AccessToken = string;
+
+/**
+ * @deprecated Use `AccessToken` instead. Pass { accessToken: "hf_..." } instead of { credentials: { accessToken: "hf_..." } }
+ */
+export interface Credentials {
+	accessToken: AccessToken;
+}
+
+export type CredentialsParams =
+	| {
+			accessToken?: undefined;
+			/**
+			 * @deprecated Use `accessToken` instead
+			 */
+			credentials: Credentials;
+	  }
+	| {
+			accessToken: AccessToken;
+			/**
+			 * @deprecated Use `accessToken` instead
+			 */
+			credentials?: undefined;
+	  };
+
+export type SpaceHardwareFlavor =
+	| "cpu-basic"
+	| "cpu-upgrade"
+	| "cpu-performance"
+	| "cpu-xl"
+	| "sprx8"
+	| "zero-a10g"
+	| "inf2x6"
+	| "t4-small"
+	| "t4-medium"
+	| "l4x1"
+	| "l4x4"
+	| "l40sx1"
+	| "l40sx4"
+	| "l40sx8"
+	| "a10g-small"
+	| "a10g-large"
+	| "a10g-largex2"
+	| "a10g-largex4"
+	| "a100-large"
+	| "a100x4"
+	| "a100x8";
+
+export type SpaceSdk = "streamlit" | "gradio" | "docker" | "static";
+
+export type SpaceStage =
+	| "NO_APP_FILE"
+	| "CONFIG_ERROR"
+	| "BUILDING"
+	| "BUILD_ERROR"
+	| "RUNNING"
+	| "RUNNING_BUILDING"
+	| "RUNTIME_ERROR"
+	| "DELETING"
+	| "PAUSED"
+	| "SLEEPING";
+
+export type AccessTokenRole = "admin" | "write" | "contributor" | "read";
+
+export type AuthType = "access_token" | "app_token" | "app_token_as_user";
+
+export type { PipelineType };
+
+export interface SpaceRuntime {
+	stage: SpaceStage;
+	sdk?: SpaceSdk;
+	sdkVersion?: string;
+	errorMessage?: string;
+	hardware?: {
+		current: SpaceHardwareFlavor | null;
+		currentPrettyName?: string;
+		requested: SpaceHardwareFlavor | null;
+		requestedPrettyName?: string;
+	};
+	/** when calling /spaces, those props are only fetched if ?full=true */
+	resources?: SpaceResourceConfig;
+	/** in seconds */
+	gcTimeout?: number | null;
+}
+
+export interface SpaceResourceRequirement {
+	cpu?: string;
+	memory?: string;
+	gpu?: string;
+	gpuModel?: string;
+	ephemeral?: string;
+}
+
+export interface SpaceResourceConfig {
+	requests: SpaceResourceRequirement;
+	limits: SpaceResourceRequirement;
+	replicas?: number;
+	throttled?: boolean;
+	is_custom?: boolean;
+}
+
+export type License =
+	| "apache-2.0"
+	| "mit"
+	| "openrail"
+	| "bigscience-openrail-m"
+	| "creativeml-openrail-m"
+	| "bigscience-bloom-rail-1.0"
+	| "bigcode-openrail-m"
+	| "afl-3.0"
+	| "artistic-2.0"
+	| "bsl-1.0"
+	| "bsd"
+	| "bsd-2-clause"
+	| "bsd-3-clause"
+	| "bsd-3-clause-clear"
+	| "c-uda"
+	| "cc"
+	| "cc0-1.0"
+	| "cc-by-2.0"
+	| "cc-by-2.5"
+	| "cc-by-3.0"
+	| "cc-by-4.0"
+	| "cc-by-sa-3.0"
+	| "cc-by-sa-4.0"
+	| "cc-by-nc-2.0"
+	| "cc-by-nc-3.0"
+	| "cc-by-nc-4.0"
+	| "cc-by-nd-4.0"
+	| "cc-by-nc-nd-3.0"
+	| "cc-by-nc-nd-4.0"
+	| "cc-by-nc-sa-2.0"
+	| "cc-by-nc-sa-3.0"
+	| "cc-by-nc-sa-4.0"
+	| "cdla-sharing-1.0"
+	| "cdla-permissive-1.0"
+	| "cdla-permissive-2.0"
+	| "wtfpl"
+	| "ecl-2.0"
+	| "epl-1.0"
+	| "epl-2.0"
+	| "etalab-2.0"
+	| "eupl-1.1"
+	| "agpl-3.0"
+	| "gfdl"
+	| "gpl"
+	| "gpl-2.0"
+	| "gpl-3.0"
+	| "lgpl"
+	| "lgpl-2.1"
+	| "lgpl-3.0"
+	| "isc"
+	| "lppl-1.3c"
+	| "ms-pl"
+	| "mpl-2.0"
+	| "odc-by"
+	| "odbl"
+	| "openrail++"
+	| "osl-3.0"
+	| "postgresql"
+	| "ofl-1.1"
+	| "ncsa"
+	| "unlicense"
+	| "zlib"
+	| "pddl"
+	| "lgpl-lr"
+	| "deepfloyd-if-license"
+	| "llama2"
+	| "llama3"
+	| "llama3.1"
+	| "llama3.2"
+	| "llama3.3"
+	| "gemma"
+	| "apple-ascl"
+	| "apple-amlr"
+	| "unknown"
+	| "other";

package/src/vendor/hfjs-xet/utils/ChunkCache.ts

@@ -0,0 +1,102 @@
+// @ts-nocheck -- vendored upstream code, kept verbatim; our strict tsconfig does not apply.
+// Vendored from huggingface/huggingface.js@f8fdf6be (packages/hub/src/utils/ChunkCache.ts), MIT License.
+// Delete this directory when bucket support is upstreamed to @huggingface/hub.
+const CHUNK_CACHE_INITIAL_SIZE = 10_000;
+const CHUNK_CACHE_GROW_FACTOR = 1.5;
+const CHUNK_CACHE_MAX_SIZE = 1_000_000;
+
+export class ChunkCache {
+	index = 0;
+	// Index >= 0 means local xorb, < 0 means remote xorb
+	xorbIndices: Int32Array;
+	// Max 8K chunks per xorb, less than 64K uint16_t
+	chunkIndices: Uint16Array;
+	map = new Map<string, number>(); // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B
+	hmacs = new Set<string>(); // todo : remove old hmacs
+	maxSize: number;
+
+	constructor(maxSize: number = CHUNK_CACHE_MAX_SIZE) {
+		if (maxSize < 1) {
+			throw new Error("maxSize must be at least 1");
+		}
+		this.maxSize = maxSize;
+		this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize));
+		this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize));
+	}
+
+	addChunkToCache(hash: string, xorbIndex: number, chunkIndex: number, hmac: string | null): void {
+		if (this.map.has(hash)) {
+			// Happens when we receive an existing chunk from remote dedup info (eg duplicate chunk in shard? Or shards with same hmac key
+			// sharing chunks/xorbs)
+
+			// processing this chunk again would desync the cache, as `this.map.size` would not increase, as opposed to `this.index`
+
+			// We could readd/remove it to "refresh it"
+			return;
+		}
+		if (this.map.values().next().value === this.index) {
+			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+			this.map.delete(this.map.keys().next().value!);
+		}
+		this.map.set(hash, this.index);
+		if (hmac !== null) {
+			this.hmacs.add(hmac);
+		}
+
+		if (this.index >= this.xorbIndices.length) {
+			// todo: switch to resize() with modern browsers
+			const oldXorbIndices = this.xorbIndices;
+			const oldChunkIndices = this.chunkIndices;
+			this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize));
+			this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize));
+			this.xorbIndices.set(oldXorbIndices);
+			this.chunkIndices.set(oldChunkIndices);
+		}
+
+		this.xorbIndices[this.index] = xorbIndex;
+		this.chunkIndices[this.index] = chunkIndex;
+		this.index = (this.index + 1) % this.maxSize;
+	}
+
+	getChunk(
+		hash: string,
+		/**
+		 * Set to null if you only want to check against locally created chunks, or the hash is already a hmac
+		 */
+		hmacFunction: ((hash: string, key: string) => string) | null,
+	):
+		| {
+				xorbIndex: number;
+				chunkIndex: number;
+		  }
+		| undefined {
+		let index = this.map.get(hash);
+		if (index === undefined && hmacFunction !== null) {
+			for (const hmac of this.hmacs) {
+				index = this.map.get(hmacFunction(hash, hmac));
+				if (index !== undefined) {
+					break;
+				}
+			}
+		}
+		if (index === undefined) {
+			return undefined;
+		}
+		return {
+			xorbIndex: this.xorbIndices[index],
+			chunkIndex: this.chunkIndices[index],
+		};
+	}
+
+	updateChunkIndex(hash: string, chunkIndex: number): void {
+		const index = this.map.get(hash);
+		if (index === undefined) {
+			throw new Error(`Chunk not found in cache: ${hash}`);
+		}
+		this.chunkIndices[index] = chunkIndex;
+	}
+
+	removeChunkFromCache(hash: string): void {
+		this.map.delete(hash);
+	}
+}

package/src/vendor/hfjs-xet/utils/RangeList.ts

@@ -0,0 +1,182 @@
+// @ts-nocheck -- vendored upstream code, kept verbatim; our strict tsconfig does not apply.
+// Vendored from huggingface/huggingface.js@f8fdf6be (packages/hub/src/utils/RangeList.ts), MIT License.
+// Delete this directory when bucket support is upstreamed to @huggingface/hub.
+/**
+ * Code generated with this prompt by Cursor:
+ *
+ * I want to build a class to manage ranges
+ *
+ * I can add ranges to it with a start& an end (both integer, end > start). It should store those ranges efficiently.
+ *
+ * When several ranges overlap, eg [1, 100] and [30, 50], I want the class to split the range into non-overlapping ranges, and add a "ref counter" to the ranges. For example,  [1, 30], [30, 50] * 2, [50, 100]
+ *
+ * I also want to be able to remove ranges, it will decrease the ref counter or remove the range altogether. I can only remove ranges at existing boundaries. For example, with the [1, 30], [30, 50] * 2, [50, 100] configuration
+ *
+ * - removing [1, 100] => the only range remaning is [30, 50]
+ * - removing [2, 50] => error, because "2' is not a boundary
+ * - removing [30, 50] => [1, 30], [30, 50], [50, 100] (do not "merge" the ranges back together)
+ *
+ * I want to be able to associate data to each range. And I want to be able to get the ranges inside boundaries. For example , with [1, 30], [30, 50] * 2, [50, 100] configuration
+ *
+ * - getting [30, 100] => I receive [30, 50] * 2, [50, 100], and I can get / modify the data associated to each range by accessing their data prop. Note the "*2" is just the ref counter, there is onlly one range object for the interval returned
+ * - getting [2, 50] => I get [30, 50] * 2
+ *
+ * ----
+ *
+ * Could optimize with binary search, but the ranges we want to handle are not that many.
+ */
+interface Range<T> {
+	start: number;
+	end: number;
+	refCount: number;
+	data: T | null;
+}
+
+export class RangeList<T> {
+	private ranges: Range<T>[] = [];
+
+	/**
+	 * Add a range to the list. If it overlaps with existing ranges,
+	 * it will split them and increment reference counts accordingly.
+	 */
+	add(start: number, end: number): void {
+		if (end <= start) {
+			throw new TypeError("End must be greater than start");
+		}
+
+		// Find all ranges that overlap with the new range
+		const overlappingRanges: { index: number; range: Range<T> }[] = [];
+		for (let i = 0; i < this.ranges.length; i++) {
+			const range = this.ranges[i];
+			if (start < range.end && end > range.start) {
+				overlappingRanges.push({ index: i, range });
+			}
+			if (range.data !== null) {
+				throw new Error("Overlapping range already has data");
+			}
+		}
+
+		if (overlappingRanges.length === 0) {
+			// No overlaps, just add the new range
+			this.ranges.push({ start, end, refCount: 1, data: null });
+			this.ranges.sort((a, b) => a.start - b.start);
+			return;
+		}
+
+		// Handle overlaps by splitting ranges
+		const newRanges: Range<T>[] = [];
+		let currentPos = start;
+
+		for (let i = 0; i < overlappingRanges.length; i++) {
+			const { range } = overlappingRanges[i];
+
+			// Add range before overlap if exists
+			if (currentPos < range.start) {
+				newRanges.push({
+					start: currentPos,
+					end: range.start,
+					refCount: 1,
+					data: null,
+				});
+			} else if (range.start < currentPos) {
+				newRanges.push({
+					start: range.start,
+					end: currentPos,
+					refCount: range.refCount,
+					data: null,
+				});
+			}
+
+			// Add overlapping part with increased ref count
+			newRanges.push({
+				start: Math.max(currentPos, range.start),
+				end: Math.min(end, range.end),
+				refCount: range.refCount + 1,
+				data: null,
+			});
+
+			// Add remaining part of existing range if exists
+			if (range.end > end) {
+				newRanges.push({
+					start: end,
+					end: range.end,
+					refCount: range.refCount,
+					data: null,
+				});
+			}
+
+			currentPos = Math.max(currentPos, range.end);
+		}
+
+		// Add remaining part after last overlap if exists
+		if (currentPos < end) {
+			newRanges.push({
+				start: currentPos,
+				end,
+				refCount: 1,
+				data: null,
+			});
+		}
+
+		// Remove old overlapping ranges and insert new ones
+		const firstIndex = overlappingRanges[0].index;
+		const lastIndex = overlappingRanges[overlappingRanges.length - 1].index;
+		this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges);
+		this.ranges.sort((a, b) => a.start - b.start);
+	}
+
+	/**
+	 * Remove a range from the list. The range must start and end at existing boundaries.
+	 */
+	remove(start: number, end: number): void {
+		if (end <= start) {
+			throw new TypeError("End must be greater than start");
+		}
+
+		// Find ranges that need to be modified
+		const affectedRanges: { index: number; range: Range<T> }[] = [];
+		for (let i = 0; i < this.ranges.length; i++) {
+			const range = this.ranges[i];
+			if (start < range.end && end > range.start) {
+				affectedRanges.push({ index: i, range });
+			}
+		}
+
+		if (affectedRanges.length === 0) {
+			throw new Error("No ranges found to remove");
+		}
+
+		// Verify boundaries match
+		if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) {
+			throw new Error("Range boundaries must match existing boundaries");
+		}
+
+		// Todo: also check if there's a gap in the middle but it should not happen with our usage
+
+		for (let i = 0; i < affectedRanges.length; i++) {
+			const { range } = affectedRanges[i];
+
+			range.refCount--;
+		}
+
+		this.ranges = this.ranges.filter((range) => range.refCount > 0);
+	}
+
+	/**
+	 * Get all ranges within the specified boundaries.
+	 */
+	getRanges(start: number, end: number): Range<T>[] {
+		if (end <= start) {
+			throw new TypeError("End must be greater than start");
+		}
+
+		return this.ranges.filter((range) => start < range.end && end > range.start);
+	}
+
+	/**
+	 * Get all ranges in the list
+	 */
+	getAllRanges(): Range<T>[] {
+		return [...this.ranges];
+	}
+}