@genai-fi/nanogpt 0.2.12 → 0.3.0

package/dist/training/DatasetBuilder.js

@@ -1,31 +1,1276 @@
-class z {
+import { a7 as $, a8 as m, a9 as M, a as R, aa as f, ab as v, ac as z, j as _, t as x } from "../index-pWA4_lUh.js";
+import { s as E } from "../index-C4L8Cm77.js";
+import { s as P } from "../stack-BMm-efee.js";
+import { t as D } from "../tensor-DJVbYhh1.js";
+import "../index-Tf7vU29b.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+function L(s, t) {
+  return I(s, t);
+}
+function I(s, t, e = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
+  if (s == null)
+    return null;
+  if (typeof Blob == "function" && s instanceof Blob)
+    return s.slice();
+  if (r.has(s))
+    throw new Error("Circular references are not supported.");
+  if (e.has(s))
+    return e.get(s);
+  const n = t(s);
+  if (n.recurse && n.value !== null)
+    throw new Error("A deep map function may not return both a value and recurse=true.");
+  if (n.recurse)
+    if (p(s)) {
+      const a = Array.isArray(s) ? [] : {};
+      r.add(s);
+      for (const l in s) {
+        const h = s[l], c = I(h, t, e, r);
+        a[l] = c;
+      }
+      return r.delete(s), s.__proto__ && (a.__proto__ = s.__proto__), a;
+    } else
+      throw new Error(`Can't recurse into non-iterable type: ${s}`);
+  else return e.set(s, n.value), n.value;
+}
+function O(s, t = T) {
+  return A(s, t);
+}
+function A(s, t, e = /* @__PURE__ */ new Set()) {
+  const r = s[0];
+  if (e.has(r))
+    throw new Error("Circular references are not supported.");
+  const n = t(s);
+  if (n.recurse && n.value !== null)
+    throw new Error("A deep zip function may not return both a value and recurse=true.");
+  if (n.recurse)
+    if (p(r)) {
+      const a = Array.isArray(r) ? [] : {};
+      e.add(r);
+      for (const l in r) {
+        const h = s.map((w) => w[l]), c = A(h, t, e);
+        a[l] = c;
+      }
+      return e.delete(r), a;
+    } else
+      throw new Error(`Can't recurse into non-iterable type: ${r}`);
+  else return n.value;
+}
+function T(s) {
+  return s === null ? null : p(s[0]) ? { value: null, recurse: !0 } : { value: s, recurse: !1 };
+}
+function p(s) {
+  let t = !1;
+  if (M().get("IS_BROWSER"))
+    t = s instanceof TextDecoder;
+  else {
+    const { StringDecoder: e } = require("string_decoder");
+    t = s instanceof e;
+  }
+  return s != null && !ArrayBuffer.isView(s) && (Array.isArray(s) || typeof s == "object" && !(s instanceof m) && !(s instanceof Promise) && !t);
+}
+function H(s) {
+  return s == null || q(s) || Array.isArray(s) || typeof s == "object" && s instanceof m || $(s);
+}
+function q(s) {
+  return s === null || typeof s != "object" && typeof s != "function";
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+function Q(s) {
+  return L(s, G);
+}
+function G(s) {
+  return s instanceof m ? { value: s.clone(), recurse: !1 } : p(s) ? { value: null, recurse: !0 } : { value: s, recurse: !1 };
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+class C {
+  /**
+   * Constructs a `RingBuffer`.
+   * @param capacity The number of items that the buffer can accomodate.
+   */
+  constructor(t) {
+    if (this.capacity = t, this.begin = 0, this.end = 0, t == null)
+      throw new RangeError("Can't create a ring buffer of unknown capacity.");
+    if (t < 1)
+      throw new RangeError("Can't create ring buffer of capacity < 1.");
+    this.data = new Array(t), this.doubledCapacity = 2 * t;
+  }
+  /**
+   * Map any index into the range 0 <= index < 2*capacity.
+   */
+  wrap(t) {
+    for (; t < 0; )
+      t += this.doubledCapacity;
+    return t % this.doubledCapacity;
+  }
+  get(t) {
+    if (t < 0)
+      throw new RangeError("Can't get item at a negative index.");
+    return this.data[t % this.capacity];
+  }
+  set(t, e) {
+    if (t < 0)
+      throw new RangeError("Can't set item at a negative index.");
+    this.data[t % this.capacity] = e;
+  }
+  /**
+   * Returns the current number of items in the buffer.
+   */
+  length() {
+    let t = this.end - this.begin;
+    return t < 0 && (t = this.doubledCapacity + t), t;
+  }
+  /**
+   * Reports whether the buffer is full.
+   * @returns true if the number of items in the buffer equals its capacity, and
+   *   false otherwise.
+   */
+  isFull() {
+    return this.length() === this.capacity;
+  }
+  /**
+   * Reports whether the buffer is empty.
+   * @returns true if the number of items in the buffer equals zero, and
+   *   false otherwise.
+   */
+  isEmpty() {
+    return this.length() === 0;
+  }
+  /**
+   * Adds an item to the end of the buffer.
+   */
+  push(t) {
+    if (this.isFull())
+      throw new RangeError("Ring buffer is full.");
+    this.set(this.end, t), this.end = this.wrap(this.end + 1);
+  }
+  /**
+   * Adds many items to the end of the buffer, in order.
+   */
+  pushAll(t) {
+    for (const e of t)
+      this.push(e);
+  }
+  /**
+   * Removes and returns the last item in the buffer.
+   */
+  pop() {
+    if (this.isEmpty())
+      throw new RangeError("Ring buffer is empty.");
+    this.end = this.wrap(this.end - 1);
+    const t = this.get(this.end);
+    return this.set(this.end, void 0), t;
+  }
+  /**
+   * Adds an item to the beginning of the buffer.
+   */
+  unshift(t) {
+    if (this.isFull())
+      throw new RangeError("Ring buffer is full.");
+    this.begin = this.wrap(this.begin - 1), this.set(this.begin, t);
+  }
+  /**
+   * Removes and returns the first item in the buffer.
+   */
+  shift() {
+    if (this.isEmpty())
+      throw new RangeError("Ring buffer is empty.");
+    const t = this.get(this.begin);
+    return this.set(this.begin, void 0), this.begin = this.wrap(this.begin + 1), t;
+  }
+  /**
+   * Removes and returns a specific item in the buffer, and moves the last item
+   * to the vacated slot.  This is useful for implementing a shuffling stream.
+   * Note that this operation necessarily scrambles the original order.
+   *
+   * @param relativeIndex: the index of the item to remove, relative to the
+   *   first item in the buffer (e.g., hiding the ring nature of the underlying
+   *   storage).
+   */
+  shuffleExcise(t) {
+    if (this.isEmpty())
+      throw new RangeError("Ring buffer is empty.");
+    const e = this.wrap(this.begin + t), r = this.get(e);
+    return this.set(e, this.pop()), r;
+  }
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+class y extends C {
+  /**
+   * Constructs a `GrowingRingBuffer`.
+   */
+  constructor() {
+    super(y.INITIAL_CAPACITY);
+  }
+  isFull() {
+    return !1;
+  }
+  push(t) {
+    super.isFull() && this.expand(), super.push(t);
+  }
+  unshift(t) {
+    super.isFull() && this.expand(), super.unshift(t);
+  }
+  /**
+   * Doubles the capacity of the buffer.
+   */
+  expand() {
+    const t = this.capacity * 2, e = new Array(t), r = this.length();
+    for (let n = 0; n < r; n++)
+      e[n] = this.get(this.wrap(this.begin + n));
+    this.data = e, this.capacity = t, this.doubledCapacity = 2 * this.capacity, this.begin = 0, this.end = r;
+  }
+}
+y.INITIAL_CAPACITY = 32;
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+function W(s) {
+  return new Y(s);
+}
+function k(s) {
+  return new J(s);
+}
+function U(s, t) {
+  return new F(s, t);
+}
+class i {
+  /**
+   * Collect all remaining elements of a bounded stream into an array.
+   * Obviously this will succeed only for small streams that fit in memory.
+   * Useful for testing.
+   *
+   * @returns A Promise for an array of stream elements, which will resolve
+   *   when the stream is exhausted.
+   */
+  async toArray() {
+    const t = [];
+    let e = await this.next();
+    for (; !e.done; )
+      t.push(e.value), e = await this.next();
+    return t;
+  }
+  /**
+   * Collect all elements of this dataset into an array with prefetching 100
+   * elements. This is useful for testing, because the prefetch changes the
+   * order in which the Promises are resolved along the processing pipeline.
+   * This may help expose bugs where results are dependent on the order of
+   * Promise resolution rather than on the logical order of the stream (i.e.,
+   * due to hidden mutable state).
+   *
+   * @returns A Promise for an array of stream elements, which will resolve
+   *   when the stream is exhausted.
+   */
+  async toArrayForTest() {
+    const t = this.prefetch(100), e = [];
+    let r = await t.next();
+    for (; !r.done; )
+      e.push(r.value), r = await t.next();
+    return e;
+  }
+  /**
+   * Draw items from the stream until it is exhausted.
+   *
+   * This can be useful when the stream has side effects but no output.  In
+   * that case, calling this function guarantees that the stream will be
+   * fully processed.
+   */
+  async resolveFully() {
+    let t = await this.next();
+    for (; !t.done; )
+      t = await this.next();
+  }
+  /**
+   * Draw items from the stream until it is exhausted, or a predicate fails.
+   *
+   * This can be useful when the stream has side effects but no output.  In
+   * that case, calling this function guarantees that the stream will be
+   * fully processed.
+   */
+  async resolveWhile(t) {
+    let e = await this.next(), r = t(e.value);
+    for (; !e.done && r; )
+      e = await this.next(), r = t(e.value);
+  }
+  /**
+   * Handles errors thrown on this stream using a provided handler function.
+   *
+   * @param handler A function that handles any `Error` thrown during a `next()`
+   *   call and returns true if the stream should continue (dropping the failed
+   *   call) or false if the stream should quietly terminate.  If the handler
+   *   itself throws (or rethrows) an `Error`, that will be propagated.
+   *
+   * @returns A `LazyIterator` of elements passed through from upstream,
+   *   possibly filtering or terminating on upstream `next()` calls that
+   *   throw an `Error`.
+   */
+  handleErrors(t) {
+    return new et(this, t);
+  }
+  // TODO(soergel): Implement reduce() etc.
+  /**
+   * Filters this stream according to `predicate`.
+   *
+   * @param predicate A function mapping a stream element to a boolean or a
+   * `Promise` for one.
+   *
+   * @returns A `LazyIterator` of elements for which the predicate was true.
+   */
+  filter(t) {
+    return new Z(this, t);
+  }
+  /**
+   * Maps this stream through a 1-to-1 transform.
+   *
+   * @param transform A function mapping a stream element to a transformed
+   *   element.
+   *
+   * @returns A `LazyIterator` of transformed elements.
+   */
+  map(t) {
+    return new tt(this, t);
+  }
+  /**
+   * Maps this stream through an async 1-to-1 transform.
+   *
+   * @param transform A function mapping a stream element to a `Promise` for a
+   *   transformed stream element.
+   *
+   * @returns A `LazyIterator` of transformed elements.
+   */
+  mapAsync(t) {
+    return new g(this, t);
+  }
+  /**
+   * Maps this stream through a 1-to-1 transform, forcing serial execution.
+   *
+   * @param transform A function mapping a stream element to a transformed
+   *   element.
+   *
+   * @returns A `LazyIterator` of transformed elements.
+   */
+  serialMapAsync(t) {
+    return new g(this, t).serial();
+  }
+  /**
+   * Maps this stream through a 1-to-many transform.
+   *
+   * @param transform A function mapping a stream element to an array of
+   *   transformed elements.
+   *
+   * @returns A `DataStream` of transformed elements.
+   */
+  flatmap(t) {
+    return new rt(this, t);
+  }
+  /**
+   * Apply a function to every element of the stream.
+   *
+   * @param f A function to apply to each stream element.
+   */
+  async forEachAsync(t) {
+    return this.map(t).resolveFully();
+  }
+  /**
+   * Apply a function to every element of the stream, forcing serial execution.
+   *
+   * @param f A function to apply to each stream element.  Should return 'true'
+   *   to indicate that the stream should continue, or 'false' to cause it to
+   *   terminate.
+   */
+  async serialForEach(t) {
+    return this.serialMapAsync(t).resolveWhile((e) => e === !0);
+  }
+  /**
+   * Groups elements into batches, represented as arrays of elements.
+   *
+   * We can think of the elements of this iterator as 'rows' (even if they are
+   * nested structures).  By the same token, consecutive values for a given
+   * key within the elements form a 'column'.  This matches the usual sense of
+   * 'row' and 'column' when processing tabular data (e.g., parsing a CSV).
+   *
+   * Thus, "Row-major" means that the resulting batch is simply a collection of
+   * rows: `[row1, row2, row3, ...]`.  This is contrast to the column-major
+   * form, which is needed for vectorized computation.
+   *
+   * @param batchSize The number of elements desired per batch.
+   * @param smallLastBatch Whether to emit the final batch when it has fewer
+   *   than batchSize elements. Default true.
+   * @returns A `LazyIterator` of batches of elements, represented as arrays
+   *   of the original element type.
+   */
+  rowMajorBatch(t, e = !0) {
+    return new j(this, t, e);
+  }
+  /**
+   * Groups elements into batches, represented in column-major form.
+   *
+   * We can think of the elements of this iterator as 'rows' (even if they are
+   * nested structures).  By the same token, consecutive values for a given
+   * key within the elements form a 'column'.  This matches the usual sense of
+   * 'row' and 'column' when processing tabular data (e.g., parsing a CSV).
+   *
+   * Thus, "column-major" means that the resulting batch is a (potentially
+   * nested) structure representing the columns.  Each column entry, then,
+   * contains a collection of the values found in that column for a range of
+   * input elements.  This representation allows for vectorized computation, in
+   * contrast to the row-major form.
+   *
+   * The inputs should all have the same nested structure (i.e., of arrays and
+   * dicts).  The result is a single object with the same nested structure,
+   * where the leaves are arrays collecting the values of the inputs at that
+   * location (or, optionally, the result of a custom function applied to those
+   * arrays).
+   *
+   * @param batchSize The number of elements desired per batch.
+   * @param smallLastBatch Whether to emit the final batch when it has fewer
+   *   than batchSize elements. Default true.
+   * @param zipFn: (optional) A function that expects an array of elements at a
+   *   single node of the object tree, and returns a `DeepMapResult`.  The
+   *   `DeepMapResult` either provides a result value for that node (i.e.,
+   *   representing the subtree), or indicates that the node should be processed
+   *   recursively.  The default zipFn recurses as far as possible and places
+   *   arrays at the leaves.
+   * @returns A `LazyIterator` of batches of elements, represented as an object
+   *   with collections at the leaves.
+   */
+  columnMajorBatch(t, e = !0, r = T) {
+    return this.rowMajorBatch(t, e).map((a) => O(a, r));
+  }
+  /**
+   * Concatenate this `LazyIterator` with another.
+   *
+   * @param iterator A `LazyIterator` to be concatenated onto this one.
+   * @param baseErrorHandler An optional function that can intercept `Error`s
+   *   raised during a `next()` call on the base stream.  This function can
+   *   decide whether the error should be propagated, whether the error should
+   *   be ignored, or whether the base stream should be terminated.
+   * @returns A `LazyIterator`.
+   */
+  concatenate(t, e) {
+    return new F(W([this, t]), e);
+  }
+  /**
+   * Limits this stream to return at most `count` items.
+   *
+   * @param count The maximum number of items to provide from the stream. If
+   * a negative or undefined value is given, the entire stream is returned
+   *   unaltered.
+   */
+  take(t) {
+    return t < 0 || t == null ? this : new K(this, t);
+  }
+  /**
+   * Skips the first `count` items in this stream.
+   *
+   * @param count The number of items to skip.  If a negative or undefined
+   * value is given, the entire stream is returned unaltered.
+   */
+  skip(t) {
+    return t < 0 || t == null ? this : new X(this, t);
+  }
+  /**
+   * Prefetch the first `bufferSize` items in this stream.
+   *
+   * Note this prefetches Promises, but makes no guarantees about when those
+   * Promises resolve.
+   *
+   * @param bufferSize: An integer specifying the number of elements to be
+   *   prefetched.
+   */
+  prefetch(t) {
+    return new S(this, t);
+  }
+  // TODO(soergel): deep sharded shuffle, where supported
+  /**
+   * Randomly shuffles the elements of this stream.
+   *
+   * @param bufferSize: An integer specifying the number of elements from
+   * this stream from which the new stream will sample.
+   * @param seed: (Optional.) An integer specifying the random seed that
+   * will be used to create the distribution.
+   */
+  shuffle(t, e) {
+    return new nt(this, t, e);
+  }
+  /**
+   * Force an iterator to execute serially: each next() call will await the
+   * prior one, so that they cannot execute concurrently.
+   */
+  serial() {
+    return new V(this);
+  }
+}
+class Y extends i {
+  constructor(t) {
+    super(), this.items = t, this.trav = 0;
+  }
+  summary() {
+    return `Array of ${this.items.length} items`;
+  }
+  async next() {
+    if (this.trav >= this.items.length)
+      return { value: null, done: !0 };
+    const t = this.items[this.trav];
+    return this.trav++, { value: Q(t), done: !1 };
+  }
+}
+class J extends i {
+  constructor(t) {
+    super(), this.nextFn = t;
+  }
+  summary() {
+    return "Function call";
+  }
+  async next() {
+    try {
+      return this.nextFn();
+    } catch (t) {
+      throw t.message = `Error thrown while iterating through a dataset: ${t.message}`, t;
+    }
+  }
+}
+class V extends i {
+  constructor(t) {
+    super(), this.upstream = t, this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Serial`;
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  async serialNext() {
+    return this.upstream.next();
+  }
+}
+class X extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.maxCount = e, this.count = 0, this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Skip`;
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  async serialNext() {
+    for (; this.count++ < this.maxCount; ) {
+      const t = await this.upstream.next();
+      if (t.done)
+        return t;
+      R(t.value);
+    }
+    return this.upstream.next();
+  }
+}
+class K extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.maxCount = e, this.count = 0;
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Take`;
+  }
+  async next() {
+    return this.count++ >= this.maxCount ? { value: null, done: !0 } : this.upstream.next();
+  }
+}
+class j extends i {
+  constructor(t, e, r = !0) {
+    super(), this.upstream = t, this.batchSize = e, this.enableSmallLastBatch = r, this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  summary() {
+    return `${this.upstream.summary()} -> RowMajorBatch`;
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  async serialNext() {
+    const t = [];
+    for (; t.length < this.batchSize; ) {
+      const e = await this.upstream.next();
+      if (e.done)
+        return this.enableSmallLastBatch && t.length > 0 ? { value: t, done: !1 } : { value: null, done: !0 };
+      t.push(e.value);
+    }
+    return { value: t, done: !1 };
+  }
+}
+class Z extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.predicate = e, this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Filter`;
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  async serialNext() {
+    for (; ; ) {
+      const t = await this.upstream.next();
+      if (t.done || this.predicate(t.value))
+        return t;
+      R(t.value);
+    }
+  }
+}
+class tt extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.transform = e;
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Map`;
+  }
+  async next() {
+    const t = await this.upstream.next();
+    if (t.done)
+      return { value: null, done: !0 };
+    const e = f(t.value), r = this.transform(t.value), n = f(r);
+    for (const a of e)
+      v(a, n) || a.dispose();
+    return { value: r, done: !1 };
+  }
+}
+class et extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.handler = e, this.count = 0, this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  summary() {
+    return `${this.upstream.summary()} -> handleErrors`;
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  async serialNext() {
+    for (; ; )
+      try {
+        return await this.upstream.next();
+      } catch (t) {
+        if (!this.handler(t))
+          return { value: null, done: !0 };
+      }
+  }
+}
+class g extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.transform = e;
+  }
+  summary() {
+    return `${this.upstream.summary()} -> AsyncMap`;
+  }
+  async next() {
+    const t = await this.upstream.next();
+    if (t.done)
+      return { value: null, done: !0 };
+    const e = f(t.value), r = await this.transform(t.value), n = f(r);
+    for (const a of e)
+      v(a, n) || a.dispose();
+    return { value: r, done: !1 };
+  }
+}
+class st extends i {
+  constructor() {
+    super(), this.outputQueue = new y(), this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  async serialNext() {
+    for (; this.outputQueue.length() === 0; )
+      if (!await this.pump())
+        return { value: null, done: !0 };
+    return { value: this.outputQueue.shift(), done: !1 };
+  }
+}
+class rt extends st {
+  constructor(t, e) {
+    super(), this.upstream = t, this.transform = e;
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Flatmap`;
+  }
+  async pump() {
+    const t = await this.upstream.next();
+    if (t.done)
+      return !1;
+    const e = f(t.value), r = this.transform(t.value), n = f(r);
+    this.outputQueue.pushAll(r);
+    for (const a of e)
+      v(a, n) || a.dispose();
+    return !0;
+  }
+}
+class F extends i {
+  constructor(t, e) {
+    super(), this.baseErrorHandler = e, this.lastRead = null, this.iterator = null, this.moreIterators = t;
+  }
+  summary() {
+    return "TODO: fill in upstream of chained summaries -> Chained";
+  }
+  async next() {
+    return this.lastRead = this.readFromChain(this.lastRead), this.lastRead;
+  }
+  async readFromChain(t) {
+    if (await t, this.iterator == null) {
+      const r = await this.moreIterators.next();
+      if (r.done)
+        return { value: null, done: !0 };
+      this.iterator = r.value, this.baseErrorHandler != null && (this.iterator = this.iterator.handleErrors(this.baseErrorHandler));
+    }
+    const e = await this.iterator.next();
+    return e.done ? (this.iterator = null, this.readFromChain(t)) : e;
+  }
+}
+var b;
+(function(s) {
+  s[s.FAIL = 0] = "FAIL", s[s.SHORTEST = 1] = "SHORTEST", s[s.LONGEST = 2] = "LONGEST";
+})(b || (b = {}));
+class S extends i {
+  constructor(t, e) {
+    super(), this.upstream = t, this.bufferSize = e, this.buffer = new C(e);
+  }
+  summary() {
+    return `${this.upstream.summary()} -> Prefetch`;
+  }
+  /**
+   * Refill the prefetch buffer.  Returns only after the buffer is full, or
+   * the upstream source is exhausted.
+   */
+  refill() {
+    for (; !this.buffer.isFull(); ) {
+      const t = this.upstream.next();
+      this.buffer.push(t);
+    }
+  }
+  next() {
+    return this.refill(), this.buffer.shift();
+  }
+}
+class nt extends S {
+  constructor(t, e, r) {
+    super(t, e), this.upstream = t, this.windowSize = e, this.upstreamExhausted = !1, this.random = E.alea(r || z().toString()), this.lastRead = Promise.resolve({ value: null, done: !1 });
+  }
+  async next() {
+    return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+  }
+  randomInt(t) {
+    return Math.floor(this.random() * t);
+  }
+  chooseIndex() {
+    return this.randomInt(this.buffer.length());
+  }
+  async serialNext() {
+    for (this.upstreamExhausted || this.refill(); !this.buffer.isEmpty(); ) {
+      const t = this.chooseIndex(), e = await this.buffer.shuffleExcise(t);
+      if (e.done)
+        this.upstreamExhausted = !0;
+      else
+        return this.refill(), e;
+    }
+    return { value: null, done: !0 };
+  }
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+class N {
+  constructor() {
+    this.size = null;
+  }
+  // TODO(soergel): Make Datasets report whether repeated iterator() calls
+  // produce the same result (e.g., reading from a file) or different results
+  // (e.g., from the webcam).  Currently we don't make this distinction but it
+  // could be important for the user to know.
+  // abstract isDeterministic(): boolean;
+  /**
+   * Groups elements into batches.
+   *
+   * It is assumed that each of the incoming dataset elements has the same
+   * structure -- i.e. the same set of keys at each location in an object
+   * hierarchy.  For each key, the resulting `Dataset` provides a batched
+   * element collecting all of the incoming values for that key.
+   *
+   *  * Incoming primitives are grouped into a 1-D Tensor.
+   *  * Incoming Tensors are grouped into a new Tensor where the 0th axis is
+   *    the batch dimension.
+   *  * Incoming arrays are converted to Tensor and then batched.
+   *  * A nested array is interpreted as an n-D Tensor, so the batched result
+   *    has n+1 dimensions.
+   *  * An array that cannot be converted to Tensor produces an error.
+   *
+   * If an array should not be batched as a unit, it should first be converted
+   * to an object with integer keys.
+   *
+   * Here are a few examples:
+   *
+   * Batch a dataset of numbers:
+   * ```js
+   * const a = tf.data.array([1, 2, 3, 4, 5, 6, 7, 8]).batch(4);
+   * await a.forEachAsync(e => e.print());
+   * ```
+   *
+   * Batch a dataset of arrays:
+   * ```js
+   * const b = tf.data.array([[1], [2], [3], [4], [5], [6], [7], [8]]).batch(4);
+   * await b.forEachAsync(e => e.print());
+   * ```
+   *
+   * Batch a dataset of objects:
+   * ```js
+   * const c = tf.data.array([{a: 1, b: 11}, {a: 2, b: 12}, {a: 3, b: 13},
+   *   {a: 4, b: 14}, {a: 5, b: 15}, {a: 6, b: 16}, {a: 7, b: 17},
+   *   {a: 8, b: 18}]).batch(4);
+   * await c.forEachAsync(e => {
+   *   console.log('{');
+   *   for(var key in e) {
+   *     console.log(key+':');
+   *     e[key].print();
+   *   }
+   *   console.log('}');
+   * })
+   * ```
+   *
+   * @param batchSize The number of elements desired per batch.
+   * @param smallLastBatch Whether to emit the final batch when it has fewer
+   *   than batchSize elements. Default true.
+   * @returns A `Dataset`, from which a stream of batches can be obtained.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  batch(t, e = !0) {
+    const r = this;
+    _(t > 0, () => `batchSize needs to be positive, but it is
+      ${t}`);
+    let n;
+    return this.size === 1 / 0 || this.size == null ? n = this.size : e ? n = Math.ceil(this.size / t) : n = Math.floor(this.size / t), o(async () => (await r.iterator()).columnMajorBatch(t, e, at), n);
+  }
+  /**
+   * Concatenates this `Dataset` with another.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3]);
+   * const b = tf.data.array([4, 5, 6]);
+   * const c = a.concatenate(b);
+   * await c.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param dataset A `Dataset` to be concatenated onto this one.
+   * @returns A `Dataset`.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  concatenate(t) {
+    const e = this;
+    let r;
+    return this.size === 1 / 0 || t.size === 1 / 0 ? r = 1 / 0 : this.size != null && t.size != null ? r = this.size + t.size : r = null, o(async () => (await e.iterator()).concatenate(await t.iterator()), r);
+  }
+  /**
+   * Filters this dataset according to `predicate`.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+   *   .filter(x => x%2 === 0);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param predicate A function mapping a dataset element to a boolean or a
+   * `Promise` for one.
+   *
+   * @returns A `Dataset` of elements for which the predicate was true.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  filter(t) {
+    const e = this;
+    let r;
+    return this.size === 1 / 0 ? r = 1 / 0 : r = null, o(async () => (await e.iterator()).filter((n) => x(() => t(n))), r);
+  }
+  /**
+   * Apply a function to every element of the dataset.
+   *
+   * After the function is applied to a dataset element, any Tensors contained
+   * within that element are disposed.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3]);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param f A function to apply to each dataset element.
+   * @returns A `Promise` that resolves after all elements have been processed.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  async forEachAsync(t) {
+    return (await this.iterator()).forEachAsync(t);
+  }
+  /**
+   * Maps this dataset through a 1-to-1 transform.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3]).map(x => x*x);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param transform A function mapping a dataset element to a transformed
+   *   dataset element.
+   *
+   * @returns A `Dataset` of transformed elements.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  map(t) {
+    const e = this;
+    return o(async () => (await e.iterator()).map((r) => x(() => t(r))), this.size);
+  }
+  /**
+   * Maps this dataset through an async 1-to-1 transform.
+   *
+   * ```js
+   * const a =
+   *  tf.data.array([1, 2, 3]).mapAsync(x => new Promise(function(resolve){
+   *    setTimeout(() => {
+   *      resolve(x * x);
+   *    }, Math.random()*1000 + 500);
+   *  }));
+   * console.log(await a.toArray());
+   * ```
+   *
+   * @param transform A function mapping a dataset element to a `Promise` for a
+   *   transformed dataset element.  This transform is responsible for disposing
+   *   any intermediate `Tensor`s, i.e. by wrapping its computation in
+   *   `tf.tidy()`; that cannot be automated here (as it is in the synchronous
+   *   `map()` case).
+   *
+   * @returns A `Dataset` of transformed elements.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  mapAsync(t) {
+    const e = this;
+    return o(async () => (await e.iterator()).mapAsync(t), this.size);
+  }
+  /**
+   *  Creates a `Dataset` that prefetches elements from this dataset.
+   *
+   * @param bufferSize: An integer specifying the number of elements to be
+   *   prefetched.
+   * @returns A `Dataset`.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  prefetch(t) {
+    if (t == null)
+      throw new RangeError("`Dataset.prefetch()` requires bufferSize to be specified.");
+    const e = this;
+    return o(async () => (await e.iterator()).prefetch(t), this.size);
+  }
+  /**
+   * Repeats this dataset `count` times.
+   *
+   * NOTE: If this dataset is a function of global state (e.g. a random number
+   * generator), then different repetitions may produce different elements.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3]).repeat(3);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param count: (Optional) An integer, representing the number of times
+   *   the dataset should be repeated. The default behavior (if `count` is
+   *   `undefined` or negative) is for the dataset be repeated indefinitely.
+   * @returns A `Dataset`.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  repeat(t) {
+    const e = this;
+    let r;
+    return this.size != null && t > 0 ? r = this.size * t : t === 0 ? r = 0 : this.size != null && (t === void 0 || t < 0) ? r = 1 / 0 : r = null, o(async () => {
+      const n = k(async () => ({ value: await e.iterator(), done: !1 }));
+      return U(n.take(t));
+    }, r);
+  }
+  /**
+   * Creates a `Dataset` that skips `count` initial elements from this dataset.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3, 4, 5, 6]).skip(3);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param count: The number of elements of this dataset that should be skipped
+   *   to form the new dataset.  If `count` is greater than the size of this
+   *   dataset, the new dataset will contain no elements.  If `count`
+   *   is `undefined` or negative, skips the entire dataset.
+   *
+   * @returns A `Dataset`.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  skip(t) {
+    const e = this;
+    let r;
+    return this.size != null && t >= 0 && this.size >= t ? r = this.size - t : this.size != null && (this.size < t || t === void 0 || t < 0) ? r = 0 : r = null, o(async () => (await e.iterator()).skip(t), r);
+  }
+  /**
+   * Pseudorandomly shuffles the elements of this dataset. This is done in a
+   * streaming manner, by sampling from a given number of prefetched elements.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3, 4, 5, 6]).shuffle(3);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param bufferSize: An integer specifying the number of elements from this
+   *   dataset from which the new dataset will sample.
+   * @param seed: (Optional) An integer specifying the random seed that will
+   *   be used to create the distribution.
+   * @param reshuffleEachIteration: (Optional) A boolean, which if true
+   *   indicates that the dataset should be pseudorandomly reshuffled each time
+   *   it is iterated over. If false, elements will be returned in the same
+   *   shuffled order on each iteration. (Defaults to `true`.)
+   * @returns A `Dataset`.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  shuffle(t, e, r = !0) {
+    if (t == null || t < 0)
+      throw this.size == null ? new RangeError("`Dataset.shuffle()` requires bufferSize to be specified.") : new RangeError(`\`Dataset.shuffle()\` requires bufferSize to be specified.  If your data fits in main memory (for regular JS objects), and/or GPU memory (for \`tf.Tensor\`s), consider setting bufferSize to the dataset size (${this.size} elements)`);
+    const n = this, a = E.alea(e || z().toString());
+    return o(async () => {
+      let l = a.int32();
+      return r && (l += a.int32()), (await n.iterator()).shuffle(t, l.toString());
+    }, this.size);
+  }
+  /**
+   * Creates a `Dataset` with at most `count` initial elements from this
+   * dataset.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3, 4, 5, 6]).take(3);
+   * await a.forEachAsync(e => console.log(e));
+   * ```
+   *
+   * @param count: The number of elements of this dataset that should be taken
+   *   to form the new dataset.  If `count` is `undefined` or negative, or if
+   *   `count` is greater than the size of this dataset, the new dataset will
+   *   contain all elements of this dataset.
+   * @returns A `Dataset`.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  take(t) {
+    const e = this;
+    let r;
+    return this.size != null && this.size > t ? r = t : this.size != null && this.size <= t ? r = this.size : r = null, o(async () => (await e.iterator()).take(t), r);
+  }
+  /**
+   * Collect all elements of this dataset into an array.
+   *
+   * Obviously this will succeed only for small datasets that fit in memory.
+   * Useful for testing and generally should be avoided if possible.
+   *
+   * ```js
+   * const a = tf.data.array([1, 2, 3, 4, 5, 6]);
+   * console.log(await a.toArray());
+   * ```
+   *
+   * @returns A Promise for an array of elements, which will resolve
+   *   when a new stream has been obtained and fully consumed.
+   *
+   * @doc {heading: 'Data', subheading: 'Classes'}
+   */
+  async toArray() {
+    if (this.size === 1 / 0)
+      throw new Error("Can not convert infinite data stream to array.");
+    return (await this.iterator()).toArray();
+  }
+  /**
+   * Collect all elements of this dataset into an array with prefetching 100
+   * elements. This is useful for testing, because the prefetch changes the
+   * order in which the Promises are resolved along the processing pipeline.
+   * This may help expose bugs where results are dependent on the order of
+   * Promise resolution rather than on the logical order of the stream (i.e.,
+   * due to hidden mutable state).
+   *
+   * @returns A Promise for an array of elements, which will resolve
+   *   when a new stream has been obtained and fully consumed.
+   */
+  async toArrayForTest() {
+    if (this.size === 1 / 0)
+      throw new Error("Can not convert infinite data stream to array.");
+    return (await this.iterator()).toArrayForTest();
+  }
+}
+N.MAX_BUFFER_SIZE = 1e4;
+function o(s, t = null) {
+  return new class extends N {
+    constructor() {
+      super(...arguments), this.size = t;
+    }
+    /*
+     * Provide a new stream of elements.  Note this will also start new streams
+     * from any underlying `Dataset`s.
+     */
+    async iterator() {
+      return s();
+    }
+  }();
+}
+function at(s) {
+  if (s === null)
+    return null;
+  const t = s[0];
+  return H(t) ? { value: it(s), recurse: !1 } : { value: null, recurse: !0 };
+}
+function it(s) {
+  if (s.length === 0)
+    throw new Error("Can't make a batch of zero elements.");
+  return s[0] instanceof m ? P(s) : D(s);
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ */
+function ut(s) {
+  return o(async () => {
+    const t = await s();
+    return k(() => t.next());
+  });
+}
+class dt {
   tokenizer;
   blockSize;
-  tf;
-  constructor(s, o, i = 128) {
-    this.tokenizer = o, this.blockSize = i, this.tf = s;
+  constructor(t, e = 128) {
+    this.tokenizer = t, this.blockSize = e;
   }
   // Create dataset from text files
-  async createTextDataset(s, o = 32, i = 0, c = 1) {
-    const r = await Promise.all(s.map((t) => this.tokenizer.encode(t))), h = this.tokenizer.eosToken >= 0, n = r.map((t) => h ? [...t, this.tokenizer.eosToken] : t).flat(), a = n.slice(
-      Math.floor(i * n.length),
-      c === 1 ? void 0 : Math.floor(c * n.length)
-    ), l = (function* () {
+  async createTextDataset(t, e = 32, r = 0, n = 1) {
+    const a = await Promise.all(t.map((u) => this.tokenizer.encode(u))), l = this.tokenizer.eosToken >= 0, h = a.map((u) => l ? [...u, this.tokenizer.eosToken] : u).flat(), c = h.slice(
+      Math.floor(r * h.length),
+      n === 1 ? void 0 : Math.floor(n * h.length)
+    ), w = (function* () {
       for (; ; ) {
-        const t = Math.floor(Math.random() * (a.length - this.blockSize - 1)), e = a.slice(t, t + this.blockSize), k = a.slice(t + 1, t + this.blockSize + 1);
-        yield { xs: e, ys: k };
+        const u = Math.floor(Math.random() * (c.length - this.blockSize - 1)), d = c.slice(u, u + this.blockSize), B = c.slice(u + 1, u + this.blockSize + 1);
+        yield { xs: d, ys: B };
       }
     }).bind(this);
-    return this.tf.data.generator(l).batch(o).map((t) => {
-      const e = t;
-      return this.tf.tidy(() => ({
-        xs: e.xs.cast("int32"),
-        ys: e.ys.cast("int32")
+    return ut(w).batch(e).map((u) => {
+      const d = u;
+      return x(() => ({
+        xs: d.xs.cast("int32"),
+        ys: d.ys.cast("int32")
         // this.tf.oneHot(batchData.ys.cast('int32'), this.tokenizer.vocabSize),
       }));
     }).prefetch(2);
   }
 }
 export {
-  z as DatasetBuilder
+  dt as DatasetBuilder
 };