cactus-react-native 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +551 -720
- package/android/src/main/java/com/cactus/Cactus.java +41 -0
- package/android/src/main/java/com/cactus/LlamaContext.java +19 -0
- package/android/src/newarch/java/com/cactus/CactusModule.java +5 -0
- package/android/src/oldarch/java/com/cactus/CactusModule.java +5 -0
- package/ios/Cactus.mm +14 -0
- package/ios/CactusContext.h +1 -0
- package/ios/CactusContext.mm +18 -0
- package/ios/cactus.xcframework/ios-arm64_x86_64-simulator/cactus.framework/cactus +0 -0
- package/ios/cactus.xcframework/tvos-arm64_x86_64-simulator/cactus.framework/cactus +0 -0
- package/lib/commonjs/NativeCactus.js +10 -0
- package/lib/commonjs/NativeCactus.js.map +1 -1
- package/lib/commonjs/chat.js +37 -0
- package/lib/commonjs/grammar.js +560 -0
- package/lib/commonjs/index.js +545 -0
- package/lib/commonjs/index.js.map +1 -1
- package/lib/commonjs/lm.js +106 -0
- package/lib/commonjs/lm.js.map +1 -1
- package/lib/commonjs/projectId.js +8 -0
- package/lib/commonjs/projectId.js.map +1 -0
- package/lib/commonjs/remote.js +153 -0
- package/lib/commonjs/remote.js.map +1 -0
- package/lib/commonjs/telemetry.js +103 -0
- package/lib/commonjs/telemetry.js.map +1 -0
- package/lib/commonjs/tools.js +79 -0
- package/lib/commonjs/tools.js.map +1 -0
- package/lib/commonjs/tts.js +32 -0
- package/lib/commonjs/tts.js.map +1 -1
- package/lib/commonjs/vlm.js +150 -0
- package/lib/commonjs/vlm.js.map +1 -0
- package/lib/module/NativeCactus.js +8 -0
- package/lib/module/NativeCactus.js.map +1 -1
- package/lib/module/chat.js +33 -0
- package/lib/module/grammar.js +553 -0
- package/lib/module/index.js +435 -0
- package/lib/module/index.js.map +1 -1
- package/lib/module/lm.js +101 -0
- package/lib/module/lm.js.map +1 -0
- package/lib/module/projectId.js +4 -0
- package/lib/module/projectId.js.map +1 -0
- package/lib/module/remote.js +144 -0
- package/lib/module/remote.js.map +1 -0
- package/lib/module/telemetry.js +98 -0
- package/lib/module/telemetry.js.map +1 -0
- package/lib/module/tools.js +73 -0
- package/lib/module/tools.js.map +1 -0
- package/lib/module/tts.js +27 -0
- package/lib/module/tts.js.map +1 -1
- package/lib/module/vlm.js +145 -0
- package/lib/module/vlm.js.map +1 -1
- package/lib/typescript/NativeCactus.d.ts +7 -0
- package/lib/typescript/NativeCactus.d.ts.map +1 -1
- package/lib/typescript/index.d.ts +3 -1
- package/lib/typescript/index.d.ts.map +1 -1
- package/lib/typescript/lm.d.ts +11 -34
- package/lib/typescript/lm.d.ts.map +1 -1
- package/lib/typescript/projectId.d.ts +2 -0
- package/lib/typescript/projectId.d.ts.map +1 -0
- package/lib/typescript/remote.d.ts +7 -0
- package/lib/typescript/remote.d.ts.map +1 -0
- package/lib/typescript/telemetry.d.ts +25 -0
- package/lib/typescript/telemetry.d.ts.map +1 -0
- package/lib/typescript/tools.d.ts +0 -3
- package/lib/typescript/tools.d.ts.map +1 -1
- package/lib/typescript/tts.d.ts.map +1 -1
- package/lib/typescript/vlm.d.ts +14 -34
- package/lib/typescript/vlm.d.ts.map +1 -1
- package/package.json +4 -4
- package/scripts/postInstall.js +33 -0
- package/src/NativeCactus.ts +7 -0
- package/src/index.ts +122 -46
- package/src/lm.ts +80 -5
- package/src/projectId.ts +1 -0
- package/src/remote.ts +175 -0
- package/src/telemetry.ts +138 -0
- package/src/tools.ts +17 -58
- package/src/vlm.ts +129 -8
- package/android/src/main/jniLibs/x86_64/libcactus.so +0 -0
- package/android/src/main/jniLibs/x86_64/libcactus_x86_64.so +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,qBAAqB,EACrB,6BAA6B,EAC7B,6BAA6B,EAC7B,wBAAwB,EACxB,aAAa,EACb,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,qBAAqB,EACrB,6BAA6B,EAC7B,6BAA6B,EAC7B,wBAAwB,EACxB,aAAa,EACb,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,gBAAgB,EACjB,MAAM,gBAAgB,CAAA;AACvB,OAAO,KAAK,EACV,+BAA+B,EAC/B,iCAAiC,EAClC,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,MAAM,WAAW,CAAA;AAC9E,OAAO,KAAK,EAAE,iBAAiB,EAAE,0BAA0B,EAAE,MAAM,QAAQ,CAAA;AAE3E,OAAO,EAAE,KAAK,EAAuB,MAAM,SAAS,CAAA;AAEpD,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,uBAAuB,EACvB,qBAAqB,EACrB,6BAA6B,EAC7B,6BAA6B,EAC7B,iBAAiB,EACjB,0BAA0B,EAC1B,wBAAwB,EACxB,uBAAuB,EAGvB,+BAA+B,EAC/B,iCAAiC,GAClC,CAAA;AAED,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAA;AACpE,cAAc,UAAU,CAAA;AA6BxB,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,wBAAwB,CAAC,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAA;CAC5D,CAAA;AAOD,MAAM,MAAM,aAAa,GAAG,IAAI,CAC9B,mBAAmB,EACnB,cAAc,GAAG,cAAc,GAAG,cAAc,CACjD,GAAG;IACF,YAAY,CAAC,EACT,KAAK,GACL,KAAK,GACL,MAAM,GACN,MAAM,GACN,MAAM,GACN,QAAQ,GACR,MAAM,GACN,MAAM,CAAA;IACV,YAAY,CAAC,EACT,KAAK,GACL,KAAK,GACL,MAAM,GACN,MAAM,GACN,MAAM,GACN,QAAQ,GACR,MAAM,GACN,MAAM,CAAA;IACV,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAA;CACzD,CAAA;AAED,MAAM,MAAM,eAAe,GAAG,qBAAqB,CAAA;AAEnD,MAAM,MAAM,wBAAwB,GAAG;IACrC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,aAAa,CAAA;IAC5C,WAAW,CAAC,EAAE;QACZ,MAAM,CAAC,EAAE,OAAO,CAAA;QAChB,MAAM,EAAE,MAAM,CAAA;KACf,CAAA;IACD,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,0BAA0B,EAAE,CAAA;IACvC,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE,wBAAwB,CAAA;CAC3C,CAAA;AACD,MAAM,MAAM,gBAAgB,GAAG,IAAI,CACjC,sBAAsB,EACtB,yBAAyB,GAAG,QAAQ,CACrC,GACC,oBAAoB,CAAA;AAEtB,MAAM,MAAM,WAAW,GAAG;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;CACd,CAAA;AAkBD,qBAAa,YAAY;IACvB,EAAE,EAAE,MAAM,CAAA;IAEV,GAAG,EAAE,OAAO,CAAQ;IAEpB,WAAW,EAAE,MAAM,CAAK;IAExB,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAA;gBAEtB,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,kBAAkB;IAOtE;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAMrE;;OAEG;IACG,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAIlB,oBAAoB,IAAI,OAAO;IAI/B,gBAAgB,IAAI,OAAO;IAKrB,gBAAgB,CACpB,QAAQ,EAAE,0BAA0B,EAAE,EACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EACxB,MAAM,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,OAAO,CAAA;QACf,eAAe,CAAC,EAAE,wBAAwB,CAAA;QAC1C,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,mBAAmB,CAAC,EAAE,MAAM,CAAA;QAC5B,WAAW,CAAC,EAAE,MAAM,CAAA;KACrB,GACA,OAAO,CAAC,wBAAwB,GAAG,MAAM,CAAC;IAiBvC,mBAAmB,CACvB,MAAM,EAAE,gBAAgB,GAAG;QAAC,KAAK,EAAE,KAAK,CAAA;KAAC,EACzC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,EACpC,cAAc,GAAE,MAAU,EAC1B,cAAc,GAAE,MAAU,GAC3B,OAAO,CAAC,sBAAsB,CAAC;IA6D1B,UAAU,CACd,MAAM,EAAE,gBAAgB,EACxB,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,GACnC,OAAO,CAAC,sBAAsB,CAAC;IAqFlC,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAIrD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAIvC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,GACvB,OAAO,CAAC,qBAAqB,CAAC;IAc3B,KAAK,CACT,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,WAAW,CAAC;IAejB,iBAAiB,CACrB,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC;IAUV,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAInC,qBAAqB,IAAI,OAAO,CACpC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CACzC;IAIK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAI9B;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GAC9C;IAAE,MAAM,EAAE,MAAM,IAAI,CAAA;CAAE,CAOxB;AAED,wBAAsB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;AAaD,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAIvE;AAWD,wBAAsB,SAAS,CAC7B,EACE,KAAK,EACL,cAAc,EAAE,YAAY,EAC5B,YAAY,EAAE,WAAW,EACzB,IAAI,EACJ,SAAS,EAAE,QAAQ,EACnB,GAAG,IAAI,EACR,EAAE,aAAa,EAChB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,GACtC,OAAO,CAAC,YAAY,CAAC,CA0DvB;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErD;AAED,eAAO,MAAM,WAAW,GAAU,QAAQ,mBAAmB,gCAE5D,CAAC;AAEF,eAAO,MAAM,cAAc,GAAU,WAAW,MAAM,EAAE,YAAY,MAAM,EAAE,SAAQ,OAAe,qBAElG,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAAU,WAAW,MAAM,qBAE1D,CAAC;AAEF,eAAO,MAAM,yBAAyB,GAAU,WAAW,MAAM,qBAEhE,CAAC;AAEF,eAAO,MAAM,wBAAwB,GAAU,WAAW,MAAM,qBAE/D,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAU,WAAW,MAAM,kBAExD,CAAC;AAEF,eAAO,MAAM,oBAAoB,GAAU,WAAW,MAAM,EAAE,QAAQ,MAAM,EAAE,YAAY,MAAM,EAAE,EAAE,QAAQ,sBAAsB,KAAG,OAAO,CAAC,sBAAsB,CAalK,CAAC;AAEF,eAAO,MAAM,WAAW,GAAU,WAAW,MAAM,EAAE,kBAAkB,MAAM,qBAE5E,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAU,WAAW,MAAM,qBAEvD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAU,WAAW,MAAM,KAAG,OAAO,CAAC,aAAa,CAEzE,CAAC;AAEF,eAAO,MAAM,2BAA2B,GAAU,WAAW,MAAM,EAAE,gBAAgB,MAAM,EAAE,aAAa,MAAM,KAAG,OAAO,CAAC,2BAA2B,CAErJ,CAAC;AAEF,eAAO,MAAM,6BAA6B,GAAU,WAAW,MAAM,EAAE,aAAa,MAAM,KAAG,OAAO,CAAC,uBAAuB,CAE3H,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAU,WAAW,MAAM,EAAE,QAAQ,MAAM,EAAE,KAAG,OAAO,CAAC,uBAAuB,CAE5G,CAAC;AAEF,eAAO,MAAM,cAAc,GAAU,WAAW,MAAM,kBAErD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAU,WAAW,MAAM,EAAE,MAAM,MAAM,EAAE,aAAa,MAAM,EAAE,KAAG,OAAO,CAAC,oBAAoB,CAMnH,CAAC;AAEF,eAAO,MAAM,aAAa,GAAU,WAAW,MAAM,KAAG,OAAO,CAAC,gBAAgB,CAE/E,CAAC;AAEF,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC"}
|
package/lib/typescript/lm.d.ts
CHANGED
|
@@ -1,41 +1,18 @@
|
|
|
1
|
-
import type { ContextParams, CactusOAICompatibleMessage, NativeCompletionResult, EmbeddingParams, NativeEmbeddingResult
|
|
1
|
+
import type { ContextParams, CompletionParams, CactusOAICompatibleMessage, NativeCompletionResult, EmbeddingParams, NativeEmbeddingResult } from './index';
|
|
2
|
+
interface CactusLMReturn {
|
|
3
|
+
lm: CactusLM | null;
|
|
4
|
+
error: Error | null;
|
|
5
|
+
}
|
|
2
6
|
export declare class CactusLM {
|
|
3
7
|
private context;
|
|
4
8
|
private constructor();
|
|
5
|
-
static init(params: ContextParams, onProgress?: (progress: number) => void): Promise<
|
|
6
|
-
completion(messages: CactusOAICompatibleMessage[],
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
top_k?: number;
|
|
11
|
-
min_p?: number;
|
|
12
|
-
penalty_repeat?: number;
|
|
13
|
-
penalty_freq?: number;
|
|
14
|
-
penalty_present?: number;
|
|
15
|
-
mirostat?: number;
|
|
16
|
-
mirostat_tau?: number;
|
|
17
|
-
mirostat_eta?: number;
|
|
18
|
-
seed?: number;
|
|
19
|
-
stop?: string[];
|
|
20
|
-
ignore_eos?: boolean;
|
|
21
|
-
chat_template?: string;
|
|
22
|
-
jinja?: boolean;
|
|
23
|
-
tools?: object;
|
|
24
|
-
parallel_tool_calls?: object;
|
|
25
|
-
tool_choice?: string;
|
|
26
|
-
response_format?: CompletionResponseFormat;
|
|
27
|
-
n_probs?: number;
|
|
28
|
-
grammar?: string;
|
|
29
|
-
grammar_lazy?: boolean;
|
|
30
|
-
grammar_triggers?: Array<{
|
|
31
|
-
type: number;
|
|
32
|
-
value: string;
|
|
33
|
-
token: number;
|
|
34
|
-
}>;
|
|
35
|
-
json_schema?: string;
|
|
36
|
-
}, callback?: (data: TokenData) => void): Promise<NativeCompletionResult>;
|
|
37
|
-
embedding(text: string, params?: EmbeddingParams): Promise<NativeEmbeddingResult>;
|
|
9
|
+
static init(params: ContextParams, onProgress?: (progress: number) => void, cactusToken?: string): Promise<CactusLMReturn>;
|
|
10
|
+
completion(messages: CactusOAICompatibleMessage[], params?: CompletionParams, callback?: (data: any) => void): Promise<NativeCompletionResult>;
|
|
11
|
+
embedding(text: string, params?: EmbeddingParams, mode?: string): Promise<NativeEmbeddingResult>;
|
|
12
|
+
private _handleLocalEmbedding;
|
|
13
|
+
private _handleRemoteEmbedding;
|
|
38
14
|
rewind(): Promise<void>;
|
|
39
15
|
release(): Promise<void>;
|
|
40
16
|
}
|
|
17
|
+
export {};
|
|
41
18
|
//# sourceMappingURL=lm.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lm.d.ts","sourceRoot":"","sources":["../../src/lm.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,aAAa,EACb,0BAA0B,EAC1B,sBAAsB,EACtB,eAAe,EACf,qBAAqB,
|
|
1
|
+
{"version":3,"file":"lm.d.ts","sourceRoot":"","sources":["../../src/lm.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,0BAA0B,EAC1B,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAIhB,UAAU,cAAc;IACtB,EAAE,EAAE,QAAQ,GAAG,IAAI,CAAA;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO;WAIM,IAAI,CACf,MAAM,EAAE,aAAa,EACrB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,cAAc,CAAC;IA4BpB,UAAU,CACd,QAAQ,EAAE,0BAA0B,EAAE,EACtC,MAAM,GAAE,gBAAqB,EAC7B,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC;IAI5B,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,eAAe,EACxB,IAAI,GAAE,MAAgB,GACrB,OAAO,CAAC,qBAAqB,CAAC;YAoCnB,qBAAqB;YAIrB,sBAAsB;IAO9B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAKvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"projectId.d.ts","sourceRoot":"","sources":["../../src/projectId.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,YAAY,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare function setCactusToken(token: string | null): void;
|
|
2
|
+
export declare function getVertexAIEmbedding(text: string): Promise<number[]>;
|
|
3
|
+
export declare function getVertexAICompletion(textPrompt: string, imageData?: string, imagePath?: string, mimeType?: string): Promise<string>;
|
|
4
|
+
export declare function getTextCompletion(prompt: string): Promise<string>;
|
|
5
|
+
export declare function getVisionCompletion(prompt: string, imagePath: string): Promise<string>;
|
|
6
|
+
export declare function getVisionCompletionFromData(prompt: string, imageData: string, mimeType?: string): Promise<string>;
|
|
7
|
+
//# sourceMappingURL=remote.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remote.d.ts","sourceRoot":"","sources":["../../src/remote.ts"],"names":[],"mappings":"AAEA,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAEzD;AAED,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAiD1E;AAED,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,MAAM,EAClB,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAmFjB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEvE;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE5F;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEvH"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export interface TelemetryParams {
|
|
2
|
+
n_gpu_layers: number | null;
|
|
3
|
+
n_ctx: number | null;
|
|
4
|
+
model: string | null;
|
|
5
|
+
}
|
|
6
|
+
interface TelemetryConfig {
|
|
7
|
+
supabaseUrl: string;
|
|
8
|
+
supabaseKey: string;
|
|
9
|
+
table?: string;
|
|
10
|
+
}
|
|
11
|
+
export declare class Telemetry {
|
|
12
|
+
private static instance;
|
|
13
|
+
private config;
|
|
14
|
+
private constructor();
|
|
15
|
+
private static getFilename;
|
|
16
|
+
static autoInit(): void;
|
|
17
|
+
static init(config: TelemetryConfig): void;
|
|
18
|
+
static track(payload: Record<string, any>, options: TelemetryParams, deviceMetadata?: Record<string, any>): void;
|
|
19
|
+
static error(error: Error, options: TelemetryParams): void;
|
|
20
|
+
private trackInternal;
|
|
21
|
+
private errorInternal;
|
|
22
|
+
private sendRecord;
|
|
23
|
+
}
|
|
24
|
+
export {};
|
|
25
|
+
//# sourceMappingURL=telemetry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/telemetry.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CACrB;AAmBD,UAAU,eAAe;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAA0B;IACjD,OAAO,CAAC,MAAM,CAA4B;IAE1C,OAAO;IAOP,OAAO,CAAC,MAAM,CAAC,WAAW;IAQ1B,MAAM,CAAC,QAAQ,IAAI,IAAI;IASvB,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;IAM1C,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAOhH,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI;IAO1D,OAAO,CAAC,aAAa;IAoBrB,OAAO,CAAC,aAAa;YAuBP,UAAU;CAYzB"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { CactusOAICompatibleMessage } from "./chat";
|
|
2
1
|
import type { NativeCompletionResult } from "./NativeCactus";
|
|
3
2
|
interface Parameter {
|
|
4
3
|
type: string;
|
|
@@ -26,13 +25,11 @@ export declare class Tools {
|
|
|
26
25
|
}[];
|
|
27
26
|
execute(name: string, args: any): Promise<any>;
|
|
28
27
|
}
|
|
29
|
-
export declare function injectToolsIntoMessages(messages: CactusOAICompatibleMessage[], tools: Tools): CactusOAICompatibleMessage[];
|
|
30
28
|
export declare function parseAndExecuteTool(result: NativeCompletionResult, tools: Tools): Promise<{
|
|
31
29
|
toolCalled: boolean;
|
|
32
30
|
toolName?: string;
|
|
33
31
|
toolInput?: any;
|
|
34
32
|
toolOutput?: any;
|
|
35
33
|
}>;
|
|
36
|
-
export declare function updateMessagesWithToolCall(messages: CactusOAICompatibleMessage[], toolName: string, toolInput: any, toolOutput: any): CactusOAICompatibleMessage[];
|
|
37
34
|
export {};
|
|
38
35
|
//# sourceMappingURL=tools.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAE7D,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AASD,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAA2B;IAExC,GAAG,CACC,IAAI,EAAE,QAAQ,EACd,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;KAAC;IAa1C,UAAU;;;;;;;;;;;;;;IAeJ,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG;CAKtC;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,GAAG,CAAC;IAAC,UAAU,CAAC,EAAE,GAAG,CAAA;CAAC,CAAC,CA6B5K"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../../src/tts.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EAKb,MAAM,SAAS,CAAA;AAChB,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,SAAS,CAAA;AAEtD,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO;WAIM,IAAI,CACf,OAAO,EAAE,YAAY,EACrB,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,SAAS,CAAC;IAKf,QAAQ,CACZ,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../../src/tts.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EAKb,MAAM,SAAS,CAAA;AAChB,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,SAAS,CAAA;AAEtD,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO;WAIM,IAAI,CACf,OAAO,EAAE,YAAY,EACrB,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,SAAS,CAAC;IAKf,QAAQ,CACZ,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,uBAAuB,CAAC;IAc7B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
package/lib/typescript/vlm.d.ts
CHANGED
|
@@ -1,44 +1,24 @@
|
|
|
1
|
-
import type { ContextParams, CactusOAICompatibleMessage, NativeCompletionResult
|
|
1
|
+
import type { ContextParams, CompletionParams, CactusOAICompatibleMessage, NativeCompletionResult } from './index';
|
|
2
|
+
interface CactusVLMReturn {
|
|
3
|
+
vlm: CactusVLM | null;
|
|
4
|
+
error: Error | null;
|
|
5
|
+
}
|
|
2
6
|
export type VLMContextParams = ContextParams & {
|
|
3
7
|
mmproj: string;
|
|
4
8
|
};
|
|
9
|
+
export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
|
|
10
|
+
images?: string[];
|
|
11
|
+
mode?: string;
|
|
12
|
+
};
|
|
5
13
|
export declare class CactusVLM {
|
|
6
14
|
private context;
|
|
7
15
|
private constructor();
|
|
8
|
-
static init(params: VLMContextParams, onProgress?: (progress: number) => void): Promise<
|
|
9
|
-
completion(messages: CactusOAICompatibleMessage[],
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
temperature?: number;
|
|
13
|
-
top_p?: number;
|
|
14
|
-
top_k?: number;
|
|
15
|
-
min_p?: number;
|
|
16
|
-
penalty_repeat?: number;
|
|
17
|
-
penalty_freq?: number;
|
|
18
|
-
penalty_present?: number;
|
|
19
|
-
mirostat?: number;
|
|
20
|
-
mirostat_tau?: number;
|
|
21
|
-
mirostat_eta?: number;
|
|
22
|
-
seed?: number;
|
|
23
|
-
stop?: string[];
|
|
24
|
-
ignore_eos?: boolean;
|
|
25
|
-
chat_template?: string;
|
|
26
|
-
jinja?: boolean;
|
|
27
|
-
tools?: object;
|
|
28
|
-
parallel_tool_calls?: object;
|
|
29
|
-
tool_choice?: string;
|
|
30
|
-
response_format?: CompletionResponseFormat;
|
|
31
|
-
n_probs?: number;
|
|
32
|
-
grammar?: string;
|
|
33
|
-
grammar_lazy?: boolean;
|
|
34
|
-
grammar_triggers?: Array<{
|
|
35
|
-
type: number;
|
|
36
|
-
value: string;
|
|
37
|
-
token: number;
|
|
38
|
-
}>;
|
|
39
|
-
json_schema?: string;
|
|
40
|
-
}, callback?: (data: TokenData) => void): Promise<NativeCompletionResult>;
|
|
16
|
+
static init(params: VLMContextParams, onProgress?: (progress: number) => void, cactusToken?: string): Promise<CactusVLMReturn>;
|
|
17
|
+
completion(messages: CactusOAICompatibleMessage[], params?: VLMCompletionParams, callback?: (data: any) => void): Promise<NativeCompletionResult>;
|
|
18
|
+
private _handleLocalCompletion;
|
|
19
|
+
private _handleRemoteCompletion;
|
|
41
20
|
rewind(): Promise<void>;
|
|
42
21
|
release(): Promise<void>;
|
|
43
22
|
}
|
|
23
|
+
export {};
|
|
44
24
|
//# sourceMappingURL=vlm.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vlm.d.ts","sourceRoot":"","sources":["../../src/vlm.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,aAAa,EACb,0BAA0B,EAC1B,sBAAsB,
|
|
1
|
+
{"version":3,"file":"vlm.d.ts","sourceRoot":"","sources":["../../src/vlm.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,0BAA0B,EAC1B,sBAAsB,EACvB,MAAM,SAAS,CAAA;AAIhB,UAAU,eAAe;IACvB,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;CACpB;AAED,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,GAAG;IACnE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO;WAIM,IAAI,CACf,MAAM,EAAE,gBAAgB,EACxB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACvC,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,eAAe,CAAC;IA8BrB,UAAU,CACd,QAAQ,EAAE,0BAA0B,EAAE,EACtC,MAAM,GAAE,mBAAwB,EAChC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC;YAuCpB,sBAAsB;YAsBtB,uBAAuB;IA+C/B,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAKvB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cactus-react-native",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Run AI models locally on mobile devices",
|
|
5
5
|
"main": "lib/commonjs/index",
|
|
6
6
|
"module": "lib/module/index",
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"src",
|
|
12
12
|
"lib",
|
|
13
|
+
"scripts",
|
|
13
14
|
"ios",
|
|
14
15
|
"ios/CMakeLists.txt",
|
|
15
16
|
"android",
|
|
@@ -31,7 +32,8 @@
|
|
|
31
32
|
"docgen": "typedoc src/index.ts --plugin typedoc-plugin-markdown --readme none --out docs/API",
|
|
32
33
|
"typecheck": "tsc --noEmit",
|
|
33
34
|
"lint": "eslint \"**/*.{js,ts,tsx}\"",
|
|
34
|
-
"release": "release-it --ci"
|
|
35
|
+
"release": "release-it --ci",
|
|
36
|
+
"postinstall": "node ./scripts/postInstall.js"
|
|
35
37
|
},
|
|
36
38
|
"keywords": [
|
|
37
39
|
"react-native",
|
|
@@ -62,7 +64,6 @@
|
|
|
62
64
|
"@types/jest": "^28.1.2",
|
|
63
65
|
"@types/react": "~17.0.21",
|
|
64
66
|
"@types/react-native": "0.70.0",
|
|
65
|
-
|
|
66
67
|
"commitlint": "^17.0.2",
|
|
67
68
|
"eslint": "^8.4.1",
|
|
68
69
|
"jest": "^28.1.1",
|
|
@@ -71,7 +72,6 @@
|
|
|
71
72
|
"react": "18.2.0",
|
|
72
73
|
"react-native": "0.72.3",
|
|
73
74
|
"react-native-builder-bob": "^0.37.0",
|
|
74
|
-
|
|
75
75
|
"release-it": "^15.0.0",
|
|
76
76
|
"typedoc": "^0.24.7",
|
|
77
77
|
"typedoc-plugin-markdown": "^3.15.3",
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const crypto = require('crypto');
|
|
4
|
+
|
|
5
|
+
const projectRoot = process.env.INIT_CWD;
|
|
6
|
+
|
|
7
|
+
if (!projectRoot || projectRoot === process.cwd()) {
|
|
8
|
+
return;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
const rootIdPath = path.join(projectRoot, '.projectId.json');
|
|
12
|
+
const internalIdPath = path.join(__dirname, '..', 'src', 'projectId.ts');
|
|
13
|
+
|
|
14
|
+
let projectId;
|
|
15
|
+
try {
|
|
16
|
+
if (fs.existsSync(rootIdPath)) {
|
|
17
|
+
projectId = JSON.parse(fs.readFileSync(rootIdPath, 'utf8'));
|
|
18
|
+
} else {
|
|
19
|
+
projectId = crypto.randomUUID();
|
|
20
|
+
fs.writeFileSync(rootIdPath, JSON.stringify(projectId), 'utf8');
|
|
21
|
+
}
|
|
22
|
+
} catch (error) {
|
|
23
|
+
console.error('Failed to read or create project ID.', error);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const fileContent = `// DO NOT EDIT. This file is automatically generated by a post-install script.\nexport const PROJECT_ID = '${projectId}';`;
|
|
28
|
+
|
|
29
|
+
try {
|
|
30
|
+
fs.writeFileSync(internalIdPath, fileContent, 'utf8');
|
|
31
|
+
} catch (error) {
|
|
32
|
+
console.error('Failed to sync project ID into the framework.', error);
|
|
33
|
+
}
|
package/src/NativeCactus.ts
CHANGED
|
@@ -306,6 +306,12 @@ export type NativeAudioDecodeResult = {
|
|
|
306
306
|
audio_data: Array<number> // Float array of audio samples
|
|
307
307
|
}
|
|
308
308
|
|
|
309
|
+
export type NativeDeviceInfo = {
|
|
310
|
+
deviceId: string
|
|
311
|
+
model: string
|
|
312
|
+
make: string
|
|
313
|
+
os: string
|
|
314
|
+
}
|
|
309
315
|
export type NativeLlamaContext = {
|
|
310
316
|
contextId: number
|
|
311
317
|
model: {
|
|
@@ -469,6 +475,7 @@ export interface Spec extends TurboModule {
|
|
|
469
475
|
contextId: number,
|
|
470
476
|
tokens: number[],
|
|
471
477
|
): Promise<NativeAudioDecodeResult>
|
|
478
|
+
getDeviceInfo(contextId: number): Promise<NativeDeviceInfo>
|
|
472
479
|
releaseVocoder(contextId: number): Promise<void>
|
|
473
480
|
|
|
474
481
|
releaseContext(contextId: number): Promise<void>
|
package/src/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
NativeAudioCompletionResult,
|
|
19
19
|
NativeAudioTokensResult,
|
|
20
20
|
NativeAudioDecodeResult,
|
|
21
|
+
NativeDeviceInfo,
|
|
21
22
|
} from './NativeCactus'
|
|
22
23
|
import type {
|
|
23
24
|
SchemaGrammarConverterPropOrder,
|
|
@@ -26,7 +27,8 @@ import type {
|
|
|
26
27
|
import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar'
|
|
27
28
|
import type { CactusMessagePart, CactusOAICompatibleMessage } from './chat'
|
|
28
29
|
import { formatChat } from './chat'
|
|
29
|
-
import { Tools,
|
|
30
|
+
import { Tools, parseAndExecuteTool } from './tools'
|
|
31
|
+
import { Telemetry, type TelemetryParams } from './telemetry'
|
|
30
32
|
export type {
|
|
31
33
|
NativeContextParams,
|
|
32
34
|
NativeLlamaContext,
|
|
@@ -50,6 +52,7 @@ export type {
|
|
|
50
52
|
}
|
|
51
53
|
|
|
52
54
|
export { SchemaGrammarConverter, convertJsonSchemaToGrammar, Tools }
|
|
55
|
+
export * from './remote'
|
|
53
56
|
|
|
54
57
|
const EVENT_ON_INIT_CONTEXT_PROGRESS = '@Cactus_onInitContextProgress'
|
|
55
58
|
const EVENT_ON_TOKEN = '@Cactus_onToken'
|
|
@@ -161,6 +164,12 @@ const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
|
|
|
161
164
|
return null
|
|
162
165
|
}
|
|
163
166
|
|
|
167
|
+
const telemetryParams: TelemetryParams = {
|
|
168
|
+
n_gpu_layers: null,
|
|
169
|
+
n_ctx: null,
|
|
170
|
+
model: null
|
|
171
|
+
}
|
|
172
|
+
|
|
164
173
|
export class LlamaContext {
|
|
165
174
|
id: number
|
|
166
175
|
|
|
@@ -231,49 +240,72 @@ export class LlamaContext {
|
|
|
231
240
|
tool_choice: params?.tool_choice,
|
|
232
241
|
})
|
|
233
242
|
}
|
|
234
|
-
|
|
243
|
+
|
|
235
244
|
async completionWithTools(
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
}
|
|
245
|
+
params: CompletionParams & {tools: Tools},
|
|
246
|
+
callback?: (data: TokenData) => void,
|
|
247
|
+
recursionCount: number = 0,
|
|
248
|
+
recursionLimit: number = 3
|
|
249
|
+
): Promise<NativeCompletionResult> {
|
|
250
|
+
if (!params.messages) { // tool calling only works with messages
|
|
251
|
+
return this.completion(params, callback);
|
|
252
|
+
}
|
|
253
|
+
if (!params.tools) { // no tools => default completion
|
|
254
|
+
return this.completion(params, callback);
|
|
255
|
+
}
|
|
256
|
+
if (recursionCount >= recursionLimit) {
|
|
257
|
+
// console.log(`Recursion limit reached (${recursionCount}/${recursionLimit}), returning default completion`)
|
|
258
|
+
return this.completion({
|
|
259
|
+
...params,
|
|
260
|
+
jinja: true,
|
|
261
|
+
tools: params.tools.getSchemas()
|
|
262
|
+
}, callback);
|
|
263
|
+
}
|
|
256
264
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
265
|
+
const messages = [...params.messages]; // avoid mutating the original messages
|
|
266
|
+
|
|
267
|
+
// console.log('Calling completion...')
|
|
268
|
+
const result = await this.completion({
|
|
269
|
+
...params,
|
|
270
|
+
messages: messages,
|
|
271
|
+
jinja: true,
|
|
272
|
+
tools: params.tools.getSchemas()
|
|
273
|
+
}, callback);
|
|
274
|
+
// console.log('Completion result:', result);
|
|
275
|
+
|
|
276
|
+
const {toolCalled, toolName, toolInput, toolOutput} =
|
|
277
|
+
await parseAndExecuteTool(result, params.tools);
|
|
278
|
+
|
|
279
|
+
if (toolCalled && toolName && toolInput) {
|
|
280
|
+
const assistantMessage = {
|
|
281
|
+
role: 'assistant',
|
|
282
|
+
content: result.content,
|
|
283
|
+
tool_calls: result.tool_calls
|
|
284
|
+
} as CactusOAICompatibleMessage;
|
|
285
|
+
|
|
286
|
+
messages.push(assistantMessage);
|
|
287
|
+
|
|
288
|
+
const toolCallId = result.tool_calls?.[0]?.id;
|
|
289
|
+
const toolMessage = {
|
|
290
|
+
role: 'tool',
|
|
291
|
+
content: JSON.stringify(toolOutput),
|
|
292
|
+
tool_call_id: toolCallId
|
|
293
|
+
} as CactusOAICompatibleMessage;
|
|
294
|
+
|
|
295
|
+
messages.push(toolMessage);
|
|
296
|
+
|
|
297
|
+
// console.log('Messages being sent to next completion:', JSON.stringify(messages, null, 2));
|
|
298
|
+
|
|
299
|
+
return await this.completionWithTools(
|
|
300
|
+
{...params, messages: messages},
|
|
301
|
+
callback,
|
|
302
|
+
recursionCount + 1,
|
|
303
|
+
recursionLimit
|
|
304
|
+
);
|
|
305
|
+
}
|
|
274
306
|
|
|
275
|
-
|
|
276
|
-
}
|
|
307
|
+
return result;
|
|
308
|
+
}
|
|
277
309
|
|
|
278
310
|
async completion(
|
|
279
311
|
params: CompletionParams,
|
|
@@ -324,12 +356,21 @@ export class LlamaContext {
|
|
|
324
356
|
if (jsonSchema) nativeParams.json_schema = JSON.stringify(jsonSchema)
|
|
325
357
|
}
|
|
326
358
|
|
|
359
|
+
const startTime = Date.now();
|
|
360
|
+
let firstTokenTime: number | null = null;
|
|
361
|
+
const deviceInfo = await getDeviceInfo(this.id);
|
|
362
|
+
|
|
363
|
+
const wrappedCallback = callback ? (data: any) => {
|
|
364
|
+
if (firstTokenTime === null) firstTokenTime = Date.now();
|
|
365
|
+
callback(data);
|
|
366
|
+
} : undefined;
|
|
367
|
+
|
|
327
368
|
let tokenListener: any =
|
|
328
|
-
|
|
369
|
+
wrappedCallback &&
|
|
329
370
|
EventEmitter.addListener(EVENT_ON_TOKEN, (evt: TokenNativeEvent) => {
|
|
330
371
|
const { contextId, tokenResult } = evt
|
|
331
372
|
if (contextId !== this.id) return
|
|
332
|
-
|
|
373
|
+
wrappedCallback(tokenResult)
|
|
333
374
|
})
|
|
334
375
|
|
|
335
376
|
if (!nativeParams.prompt) throw new Error('Prompt is required')
|
|
@@ -337,6 +378,12 @@ export class LlamaContext {
|
|
|
337
378
|
const promise = Cactus.completion(this.id, nativeParams)
|
|
338
379
|
return promise
|
|
339
380
|
.then((completionResult) => {
|
|
381
|
+
Telemetry.track({
|
|
382
|
+
event: 'completion',
|
|
383
|
+
tok_per_sec: (completionResult as any).timings?.predicted_per_second,
|
|
384
|
+
toks_generated: (completionResult as any).timings?.predicted_n,
|
|
385
|
+
ttft: firstTokenTime ? firstTokenTime - startTime : null,
|
|
386
|
+
}, telemetryParams, deviceInfo);
|
|
340
387
|
tokenListener?.remove()
|
|
341
388
|
tokenListener = null
|
|
342
389
|
return completionResult
|
|
@@ -360,11 +407,21 @@ export class LlamaContext {
|
|
|
360
407
|
return Cactus.detokenize(this.id, tokens)
|
|
361
408
|
}
|
|
362
409
|
|
|
363
|
-
embedding(
|
|
410
|
+
async embedding(
|
|
364
411
|
text: string,
|
|
365
412
|
params?: EmbeddingParams,
|
|
366
413
|
): Promise<NativeEmbeddingResult> {
|
|
367
|
-
|
|
414
|
+
const startTime = Date.now();
|
|
415
|
+
const embeddingResult = await Cactus.embedding(this.id, text, params || {})
|
|
416
|
+
const totalTime = Date.now() - startTime;
|
|
417
|
+
const deviceInfo = await getDeviceInfo(this.id);
|
|
418
|
+
Telemetry.track({
|
|
419
|
+
event: 'embedding',
|
|
420
|
+
mode: 'local',
|
|
421
|
+
embedding_time: totalTime,
|
|
422
|
+
}, telemetryParams, deviceInfo);
|
|
423
|
+
return embeddingResult;
|
|
424
|
+
|
|
368
425
|
}
|
|
369
426
|
|
|
370
427
|
async bench(
|
|
@@ -488,6 +545,10 @@ export async function initLlama(
|
|
|
488
545
|
scaled: l.scaled,
|
|
489
546
|
}))
|
|
490
547
|
|
|
548
|
+
telemetryParams.n_gpu_layers = rest.n_gpu_layers || null;
|
|
549
|
+
telemetryParams.n_ctx = rest.n_ctx || null;
|
|
550
|
+
telemetryParams.model = model;
|
|
551
|
+
|
|
491
552
|
const contextId = contextIdCounter + contextIdRandom()
|
|
492
553
|
contextIdCounter += 1
|
|
493
554
|
|
|
@@ -559,7 +620,18 @@ export const releaseMultimodal = async (contextId: number) => {
|
|
|
559
620
|
};
|
|
560
621
|
|
|
561
622
|
export const multimodalCompletion = async (contextId: number, prompt: string, mediaPaths: string[], params: NativeCompletionParams): Promise<NativeCompletionResult> => {
|
|
562
|
-
|
|
623
|
+
const result = await Cactus.multimodalCompletion(contextId, prompt, mediaPaths, params);
|
|
624
|
+
|
|
625
|
+
const deviceInfo = await getDeviceInfo(contextId);
|
|
626
|
+
|
|
627
|
+
Telemetry.track({
|
|
628
|
+
event: 'completion',
|
|
629
|
+
tok_per_sec: (result as any).timings?.predicted_per_second,
|
|
630
|
+
toks_generated: (result as any).timings?.predicted_n,
|
|
631
|
+
num_images: mediaPaths?.length,
|
|
632
|
+
}, telemetryParams, deviceInfo);
|
|
633
|
+
|
|
634
|
+
return result;
|
|
563
635
|
};
|
|
564
636
|
|
|
565
637
|
export const initVocoder = async (contextId: number, vocoderModelPath: string) => {
|
|
@@ -598,6 +670,10 @@ export const tokenize = async (contextId: number, text: string, mediaPaths?: str
|
|
|
598
670
|
}
|
|
599
671
|
};
|
|
600
672
|
|
|
673
|
+
export const getDeviceInfo = async (contextId: number): Promise<NativeDeviceInfo> => {
|
|
674
|
+
return await Cactus.getDeviceInfo(contextId);
|
|
675
|
+
};
|
|
676
|
+
|
|
601
677
|
export { CactusLM } from './lm';
|
|
602
678
|
export { CactusVLM } from './vlm';
|
|
603
679
|
export { CactusTTS } from './tts';
|