@tagma/completion-llm-judge 0.2.3 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.ts +10 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -5
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/index.test.ts +6 -5
- package/src/index.ts +10 -5
package/README.md
CHANGED
|
@@ -58,10 +58,10 @@ completion:
|
|
|
58
58
|
Or load it programmatically:
|
|
59
59
|
|
|
60
60
|
```ts
|
|
61
|
-
import {
|
|
61
|
+
import { createTagma } from '@tagma/sdk';
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
await loadPlugins(['@tagma/completion-llm-judge']);
|
|
63
|
+
const tagma = createTagma();
|
|
64
|
+
await tagma.registry.loadPlugins(['@tagma/completion-llm-judge'], process.cwd());
|
|
65
65
|
```
|
|
66
66
|
|
|
67
67
|
## Config
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { type CompletionPlugin } from '@tagma/types';
|
|
2
|
-
declare const LlmJudgeCompletion: CompletionPlugin;
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
export declare const LlmJudgeCompletion: CompletionPlugin;
|
|
3
|
+
declare const _default: {
|
|
4
|
+
name: string;
|
|
5
|
+
capabilities: {
|
|
6
|
+
completions: {
|
|
7
|
+
llm_judge: CompletionPlugin;
|
|
8
|
+
};
|
|
9
|
+
};
|
|
10
|
+
};
|
|
11
|
+
export default _default;
|
|
6
12
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA6BA,OAAO,EAGL,KAAK,gBAAgB,EAGtB,MAAM,cAAc,CAAC;AAuHtB,eAAO,MAAM,kBAAkB,EAAE,gBAmHhC,CAAC;;;;;;;;;AAEF,wBAOwB"}
|
package/dist/index.js
CHANGED
|
@@ -122,7 +122,7 @@ async function callJudge(endpoint, model, apiKey, messages, timeoutMs, externalS
|
|
|
122
122
|
externalSignal.removeEventListener('abort', onExternalAbort);
|
|
123
123
|
}
|
|
124
124
|
}
|
|
125
|
-
const LlmJudgeCompletion = {
|
|
125
|
+
export const LlmJudgeCompletion = {
|
|
126
126
|
name: 'llm_judge',
|
|
127
127
|
schema: {
|
|
128
128
|
description: 'Use an LLM to judge whether the task output satisfies a rubric. Answers PASS/FAIL.',
|
|
@@ -220,8 +220,12 @@ const LlmJudgeCompletion = {
|
|
|
220
220
|
}
|
|
221
221
|
},
|
|
222
222
|
};
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
223
|
+
export default {
|
|
224
|
+
name: '@tagma/completion-llm-judge',
|
|
225
|
+
capabilities: {
|
|
226
|
+
completions: {
|
|
227
|
+
llm_judge: LlmJudgeCompletion,
|
|
228
|
+
},
|
|
229
|
+
},
|
|
230
|
+
};
|
|
227
231
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,iBAAiB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,yCAAyC;AACzC,EAAE;AACF,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,qEAAqE;AACrE,uEAAuE;AACvE,sCAAsC;AACtC,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,uEAAuE;AACvE,8DAA8D;AAC9D,EAAE;AACF,0BAA0B;AAC1B,6CAA6C;AAC7C,YAAY;AACZ,eAAe;AACf,sBAAsB;AACtB,wBAAwB;AACxB,8BAA8B;AAC9B,mFAAmF;AACnF,sFAAsF;AAEtF,OAAO,EACL,iBAAiB,GAKlB,MAAM,cAAc,CAAC;AAEtB,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,0EAA0E;AAC1E,oEAAoE;AACpE,wEAAwE;AACxE,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAEvC,MAAM,aAAa,GACjB,oDAAoD;IACpD,oGAAoG;IACpG,qEAAqE;IACrE,uEAAuE;IACvE,uGAAuG,CAAC;AAa1G,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,2EAA2E;AAC3E,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,uBAAuB,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,oEAAoE;AACpE,oEAAoE;AACpE,oCAAoC;AACpC,SAAS,aAAa,CAAC,OAAe;IACpC,OAAO,OAAO;SACX,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC;SACzC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,QAAgB,EAChB,KAAa,EACb,MAA0B,EAC1B,QAAgC,EAChC,SAAiB,EACjB,cAAuC;IAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,MAAM,eAAe,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,gBAAgB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,qEAAqE;QACrE,mEAAmE;QACnE,sEAAsE;QACtE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,aAAa,GAAG,UAAU,MAAM,EAAE,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC;gBACd,kEAAkE;gBAClE,kEAAkE;gBAClE,kBAAkB;gBAClB,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACvD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,cAAc;YAAE,cAAc,CAAC,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACnF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAqB;IAClD,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE;QACN,WAAW,EACT,oFAAoF;QACtF,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE,4DAA4D;aAC1E;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,aAAa;gBACtB,WAAW,EACT,oJAAoJ;gBACtJ,WAAW,EAAE,aAAa;aAC3B;YACD,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,gBAAgB;gBACzB,WAAW,EACT,qHAAqH;gBACvH,WAAW,EAAE,gBAAgB;aAC9B;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EACT,2IAA2I;gBAC7I,WAAW,EAAE,gBAAgB;aAC9B;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,MAAM;gBACf,WAAW,EACT,gGAAgG;aACnG;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,wBAAwB;gBACjC,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,OAAO;gBACZ,WAAW,EAAE,yDAAyD;aACvE;SACF;KACF;IAED,KAAK,CAAC,KAAK,CACT,MAA+B,EAC/B,MAAkB,EAClB,GAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAA4B,CAAC;QACnD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAE3E,+DAA+D;QAC/D,qEAAqE;QACrE,kEAAkE;QAClE,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAiC,CAAC;QAC3D,IAAI,MAA0B,CAAC;QAC/B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,aAAa,CAAC;QACpE,MAAM,QAAQ,GAAI,MAAM,CAAC,QAA+B,IAAI,gBAAgB,CAAC;QAC7E,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACxE,MAAM,QAAQ,GACZ,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,IAAI,MAAM,CAAC,gBAAgB,GAAG,CAAC;YACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC;YACrC,CAAC,CAAC,wBAAwB,CAAC;QAE/B,oEAAoE;QACpE,qEAAqE;QACrE,oEAAoE;QACpE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;QAE5D,MAAM,WAAW,GACf,aAAa,MAAM,MAAM;YACzB,gBAAgB,MAAM,CAAC,QAAQ,MAAM;YACrC,kBAAkB,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QAE7D,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,kEAAkE;gBAClE,mEAAmE;gBACnE,OAAO,CAAC,IAAI,CACV,uBAAuB,SAAS,IAAI,SAAS,4BAA4B,OAAO,EAAE,CACnF,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,iEAAiE;YACjE,iDAAiD;YACjD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,gEAAgE,GAAG,EAAE,CAAC,CAAC;YACpF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAC;AAEF,eAAe;IACb,IAAI,EAAE,6BAA6B;IACnC,YAAY,EAAE;QACZ,WAAW,EAAE;YACX,SAAS,EAAE,kBAAkB;SAC9B;KACF;CACoB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tagma/completion-llm-judge",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.6",
|
|
4
4
|
"description": "LLM-as-judge completion plugin for tagma-sdk pipelines",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -50,6 +50,6 @@
|
|
|
50
50
|
"prepublishOnly": "bun run build"
|
|
51
51
|
},
|
|
52
52
|
"peerDependencies": {
|
|
53
|
-
"@tagma/types": "0.4.
|
|
53
|
+
"@tagma/types": "0.4.11"
|
|
54
54
|
}
|
|
55
55
|
}
|
package/src/index.test.ts
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
|
-
import plugin from './index';
|
|
2
|
+
import plugin, { LlmJudgeCompletion } from './index';
|
|
3
3
|
import manifest from '../package.json' with { type: 'json' };
|
|
4
4
|
|
|
5
5
|
describe('completion-llm-judge plugin shape', () => {
|
|
6
|
-
test('
|
|
6
|
+
test('default export is a capability plugin matching package manifest', () => {
|
|
7
7
|
expect(manifest.tagmaPlugin.category).toBe('completions');
|
|
8
8
|
expect(manifest.tagmaPlugin.type).toBe('llm_judge');
|
|
9
|
-
expect(plugin.name).toBe(manifest.
|
|
9
|
+
expect(plugin.name).toBe(manifest.name);
|
|
10
|
+
expect(plugin.capabilities?.completions?.[manifest.tagmaPlugin.type]).toBe(LlmJudgeCompletion);
|
|
10
11
|
});
|
|
11
12
|
|
|
12
|
-
test('
|
|
13
|
-
expect(typeof plugin.check).toBe('function');
|
|
13
|
+
test('check is a function', () => {
|
|
14
|
+
expect(typeof plugin.capabilities!.completions!.llm_judge.check).toBe('function');
|
|
14
15
|
});
|
|
15
16
|
});
|
package/src/index.ts
CHANGED
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import {
|
|
31
31
|
parseDurationSafe,
|
|
32
|
+
type TagmaPlugin,
|
|
32
33
|
type CompletionPlugin,
|
|
33
34
|
type CompletionContext,
|
|
34
35
|
type TaskResult,
|
|
@@ -151,7 +152,7 @@ async function callJudge(
|
|
|
151
152
|
}
|
|
152
153
|
}
|
|
153
154
|
|
|
154
|
-
const LlmJudgeCompletion: CompletionPlugin = {
|
|
155
|
+
export const LlmJudgeCompletion: CompletionPlugin = {
|
|
155
156
|
name: 'llm_judge',
|
|
156
157
|
schema: {
|
|
157
158
|
description:
|
|
@@ -268,7 +269,11 @@ const LlmJudgeCompletion: CompletionPlugin = {
|
|
|
268
269
|
},
|
|
269
270
|
};
|
|
270
271
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
272
|
+
export default {
|
|
273
|
+
name: '@tagma/completion-llm-judge',
|
|
274
|
+
capabilities: {
|
|
275
|
+
completions: {
|
|
276
|
+
llm_judge: LlmJudgeCompletion,
|
|
277
|
+
},
|
|
278
|
+
},
|
|
279
|
+
} satisfies TagmaPlugin;
|