@effect/ai-anthropic 0.0.1

package/dist/cjs/AnthropicTokenizer.js

@@ -0,0 +1,50 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.layer = void 0;
+var _tokenizer = require("@anthropic-ai/tokenizer");
+var _AiError = require("@effect/ai/AiError");
+var Tokenizer = _interopRequireWildcard(require("@effect/ai/Tokenizer"));
+var Arr = _interopRequireWildcard(require("effect/Array"));
+var Chunk = _interopRequireWildcard(require("effect/Chunk"));
+var Effect = _interopRequireWildcard(require("effect/Effect"));
+var Layer = _interopRequireWildcard(require("effect/Layer"));
+var Option = _interopRequireWildcard(require("effect/Option"));
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
+/**
+ * @since 1.0.0
+ */
+
+const make = /*#__PURE__*/Tokenizer.make({
+  tokenize(content) {
+    return Effect.try({
+      try: () => {
+        const tokenizer = (0, _tokenizer.getTokenizer)();
+        const text = Arr.flatMap(Chunk.toReadonlyArray(content), message => Arr.filterMap(message.parts, part => {
+          if (part._tag === "Image" || part._tag === "ImageUrl") {
+            return Option.none();
+          }
+          return Option.some(part._tag === "Text" ? part.content : JSON.stringify(part._tag === "ToolCall" ? part.params : part.value));
+        })).join("");
+        const encoded = tokenizer.encode(text.normalize("NFKC"), "all");
+        tokenizer.free();
+        return Array.from(encoded);
+      },
+      catch: cause => new _AiError.AiError({
+        module: "OpenAiCompletions",
+        method: "tokenize",
+        description: "Could not tokenize",
+        cause
+      })
+    });
+  }
+});
+/**
+ * @since 1.0.0
+ * @category layers
+ */
+const layer = exports.layer = /*#__PURE__*/Layer.succeed(Tokenizer.Tokenizer, make);
+//# sourceMappingURL=AnthropicTokenizer.js.map

package/dist/cjs/AnthropicTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AnthropicTokenizer.js","names":["_tokenizer","require","_AiError","Tokenizer","_interopRequireWildcard","Arr","Chunk","Effect","Layer","Option","_getRequireWildcardCache","e","WeakMap","r","t","__esModule","default","has","get","n","__proto__","a","Object","defineProperty","getOwnPropertyDescriptor","u","hasOwnProperty","call","i","set","make","tokenize","content","try","tokenizer","getTokenizer","text","flatMap","toReadonlyArray","message","filterMap","parts","part","_tag","none","some","JSON","stringify","params","value","join","encoded","encode","normalize","free","Array","from","catch","cause","AiError","module","method","description","layer","exports","succeed"],"sources":["../../src/AnthropicTokenizer.ts"],"sourcesContent":[null],"mappings":";;;;;;AAGA,IAAAA,UAAA,GAAAC,OAAA;AACA,IAAAC,QAAA,GAAAD,OAAA;AACA,IAAAE,SAAA,GAAAC,uBAAA,CAAAH,OAAA;AACA,IAAAI,GAAA,GAAAD,uBAAA,CAAAH,OAAA;AACA,IAAAK,KAAA,GAAAF,uBAAA,CAAAH,OAAA;AACA,IAAAM,MAAA,GAAAH,uBAAA,CAAAH,OAAA;AACA,IAAAO,KAAA,GAAAJ,uBAAA,CAAAH,OAAA;AACA,IAAAQ,MAAA,GAAAL,uBAAA,CAAAH,OAAA;AAAuC,SAAAS,yBAAAC,CAAA,6BAAAC,OAAA,mBAAAC,CAAA,OAAAD,OAAA,IAAAE,CAAA,OAAAF,OAAA,YAAAF,wBAAA,YAAAA,CAAAC,CAAA,WAAAA,CAAA,GAAAG,CAAA,GAAAD,CAAA,KAAAF,CAAA;AAAA,SAAAP,wBAAAO,CAAA,EAAAE,CAAA,SAAAA,CAAA,IAAAF,CAAA,IAAAA,CAAA,CAAAI,UAAA,SAAAJ,CAAA,eAAAA,CAAA,uBAAAA,CAAA,yBAAAA,CAAA,WAAAK,OAAA,EAAAL,CAAA,QAAAG,CAAA,GAAAJ,wBAAA,CAAAG,CAAA,OAAAC,CAAA,IAAAA,CAAA,CAAAG,GAAA,CAAAN,CAAA,UAAAG,CAAA,CAAAI,GAAA,CAAAP,CAAA,OAAAQ,CAAA,KAAAC,SAAA,UAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,IAAAD,MAAA,CAAAE,wBAAA,WAAAC,CAAA,IAAAd,CAAA,oBAAAc,CAAA,OAAAC,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAc,CAAA,SAAAG,CAAA,GAAAP,CAAA,GAAAC,MAAA,CAAAE,wBAAA,CAAAb,CAAA,EAAAc,CAAA,UAAAG,CAAA,KAAAA,CAAA,CAAAV,GAAA,IAAAU,CAAA,CAAAC,GAAA,IAAAP,MAAA,CAAAC,cAAA,CAAAJ,CAAA,EAAAM,CAAA,EAAAG,CAAA,IAAAT,CAAA,CAAAM,CAAA,IAAAd,CAAA,CAAAc,CAAA,YAAAN,CAAA,CAAAH,OAAA,GAAAL,CAAA,EAAAG,CAAA,IAAAA,CAAA,CAAAe,GAAA,CAAAlB,CAAA,EAAAQ,CAAA,GAAAA,CAAA;AAVvC;;;;AAYA,MAAMW,IAAI,gBAAG3B,SAAS,CAAC2B,IAAI,CAAC;EAC1BC,QAAQA,CAACC,OAAO;IACd,OAAOzB,MAAM,CAAC0B,GAAG,CAAC;MAChBA,GAAG,EAAEA,CAAA,KAAK;QACR,MAAMC,SAAS,GAAG,IAAAC,uBAAY,GAAE;QAChC,MAAMC,IAAI,GAAG/B,GAAG,CAACgC,OAAO,CAAC/B,KAAK,CAACgC,eAAe,CAACN,OAAO,CAAC,EAAGO,OAAO,IAC/DlC,GAAG,CAACmC,SAAS,CAACD,OAAO,CAACE,KAAK,EAAGC,IAAI,IAAI;UACpC,IAAIA,IAAI,CAACC,IAAI,KAAK,OAAO,IAAID,IAAI,CAACC,IAAI,KAAK,UAAU,EAAE;YACrD,OAAOlC,MAAM,CAACmC,IAAI,EAAE;UACtB;UACA,OAAOnC,MAAM,CAACoC,IAAI,CAChBH,IAAI,CAACC,IAAI,KAAK,MAAM,GAChBD,IAAI,CAACV,OAAO,GACZc,IAAI,CAACC,SAAS,CACdL,IAAI,CAACC,IAAI,KAAK,UAAU,GACpBD,IAAI,CAACM,MAAM,GACbN,IAAI,CAACO,KAAK,CACb,CACJ;QACH,CAAC,CAAC,CAAC,CAACC,IAAI,CAAC,EAAE,CAAC;QACd,MAAMC,OAAO,GAAGjB,SAAS,CAACkB,MAAM,CAAChB,IAAI,CAACiB,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC;QAC/DnB,SAAS,CAACoB,IAAI,EAAE;QAChB,OAAOC,KAAK,CAACC,IAAI,CAACL,OAAO,CAAC;MAC5B,CAAC;MACDM,KAAK,EAAGC,KAAK,IACX,IAAIC,gBAAO,CAAC;QACVC,MAAM,EAAE,mBAAmB;QAC3BC,MAAM,EAAE,UAAU;QAClBC,WAAW,EAAE,oBAAoB;QACjCJ;OACD;KACJ,CAAC;EACJ;CACD,CAAC;AAEF;;;;AAIO,MAAMK,KAAK,GAAAC,OAAA,CAAAD,KAAA,gBAAqCvD,KAAK,CAACyD,OAAO,CAAC9D,SAAS,CAACA,SAAS,EAAE2B,IAAI,CAAC","ignoreList":[]}