@devflow-tools/server 0.12.4 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/benchmark.controller.d.ts +4 -18
- package/dist/benchmark/benchmark.controller.d.ts.map +1 -1
- package/dist/benchmark/benchmark.controller.js +1 -29
- package/dist/benchmark/benchmark.controller.js.map +1 -1
- package/dist/benchmark/benchmark.service.d.ts +4 -18
- package/dist/benchmark/benchmark.service.d.ts.map +1 -1
- package/dist/benchmark/benchmark.service.js +3 -46
- package/dist/benchmark/benchmark.service.js.map +1 -1
- package/dist/workflow/workflow.controller.d.ts +4 -4
- package/dist/workflow/workflow.service.d.ts +4 -4
- package/package.json +11 -11
|
@@ -2,24 +2,10 @@ import { BenchmarkService } from "./benchmark.service.js";
|
|
|
2
2
|
export declare class BenchmarkController {
|
|
3
3
|
private readonly service;
|
|
4
4
|
constructor(service: BenchmarkService);
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
getComparisonSummary(): Promise<{
|
|
10
|
-
totalScenarios: number;
|
|
11
|
-
avgTimeSavedRate: number;
|
|
12
|
-
avgTokenReductionRate: number;
|
|
13
|
-
avgFirstPassLift: number;
|
|
14
|
-
avgReworkReductionRate: number;
|
|
15
|
-
}>;
|
|
16
|
-
getExperimentalRoiSummary(): Promise<{
|
|
17
|
-
totalScenarios: number;
|
|
18
|
-
avgTimeSavedRate: number;
|
|
19
|
-
avgTokenReductionRate: number;
|
|
20
|
-
avgFirstPassLift: number;
|
|
21
|
-
avgReworkReductionRate: number;
|
|
22
|
-
conclusion: string;
|
|
5
|
+
getResults(): Promise<{
|
|
6
|
+
results: never[];
|
|
7
|
+
isRealData: boolean;
|
|
8
|
+
message: string;
|
|
23
9
|
}>;
|
|
24
10
|
}
|
|
25
11
|
//# sourceMappingURL=benchmark.controller.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.controller.d.ts","sourceRoot":"","sources":["../../src/benchmark/benchmark.controller.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,qBACa,mBAAmB;IACQ,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,gBAAgB;IAG1E,
|
|
1
|
+
{"version":3,"file":"benchmark.controller.d.ts","sourceRoot":"","sources":["../../src/benchmark/benchmark.controller.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,qBACa,mBAAmB;IACQ,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,gBAAgB;IAG1E,UAAU;;;;;CAGjB"}
|
|
@@ -10,50 +10,22 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
10
10
|
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
11
11
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
12
12
|
};
|
|
13
|
-
import { Controller, Get,
|
|
13
|
+
import { Controller, Get, Inject } from "@nestjs/common";
|
|
14
14
|
import { BenchmarkService } from "./benchmark.service.js";
|
|
15
15
|
let BenchmarkController = class BenchmarkController {
|
|
16
16
|
constructor(service) {
|
|
17
17
|
this.service = service;
|
|
18
18
|
}
|
|
19
|
-
async run(body) {
|
|
20
|
-
return this.service.runScenarioSet(body.scenarioIds);
|
|
21
|
-
}
|
|
22
19
|
async getResults() {
|
|
23
20
|
return this.service.getScenarioResults();
|
|
24
21
|
}
|
|
25
|
-
async getComparisonSummary() {
|
|
26
|
-
return this.service.getComparisonSummary();
|
|
27
|
-
}
|
|
28
|
-
async getExperimentalRoiSummary() {
|
|
29
|
-
return this.service.getExperimentalRoiSummary();
|
|
30
|
-
}
|
|
31
22
|
};
|
|
32
|
-
__decorate([
|
|
33
|
-
Post("run"),
|
|
34
|
-
__param(0, Body()),
|
|
35
|
-
__metadata("design:type", Function),
|
|
36
|
-
__metadata("design:paramtypes", [Object]),
|
|
37
|
-
__metadata("design:returntype", Promise)
|
|
38
|
-
], BenchmarkController.prototype, "run", null);
|
|
39
23
|
__decorate([
|
|
40
24
|
Get("results"),
|
|
41
25
|
__metadata("design:type", Function),
|
|
42
26
|
__metadata("design:paramtypes", []),
|
|
43
27
|
__metadata("design:returntype", Promise)
|
|
44
28
|
], BenchmarkController.prototype, "getResults", null);
|
|
45
|
-
__decorate([
|
|
46
|
-
Get("summary"),
|
|
47
|
-
__metadata("design:type", Function),
|
|
48
|
-
__metadata("design:paramtypes", []),
|
|
49
|
-
__metadata("design:returntype", Promise)
|
|
50
|
-
], BenchmarkController.prototype, "getComparisonSummary", null);
|
|
51
|
-
__decorate([
|
|
52
|
-
Get("roi-summary"),
|
|
53
|
-
__metadata("design:type", Function),
|
|
54
|
-
__metadata("design:paramtypes", []),
|
|
55
|
-
__metadata("design:returntype", Promise)
|
|
56
|
-
], BenchmarkController.prototype, "getExperimentalRoiSummary", null);
|
|
57
29
|
BenchmarkController = __decorate([
|
|
58
30
|
Controller("benchmark"),
|
|
59
31
|
__param(0, Inject(BenchmarkService)),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.controller.js","sourceRoot":"","sources":["../../src/benchmark/benchmark.controller.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,
|
|
1
|
+
{"version":3,"file":"benchmark.controller.js","sourceRoot":"","sources":["../../src/benchmark/benchmark.controller.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAGnD,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAC9B,YAAuD,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;IAAG,CAAC;IAG9E,AAAN,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAC3C,CAAC;CACF,CAAA;AAHO;IADL,GAAG,CAAC,SAAS,CAAC;;;;qDAGd;AANU,mBAAmB;IAD/B,UAAU,CAAC,WAAW,CAAC;IAET,WAAA,MAAM,CAAC,gBAAgB,CAAC,CAAA;qCAA2B,gBAAgB;GADrE,mBAAmB,CAO/B"}
|
|
@@ -1,26 +1,12 @@
|
|
|
1
|
-
import type { BenchmarkComparisonResult } from '@devflow-tools/sdk';
|
|
2
1
|
export declare class BenchmarkService {
|
|
3
2
|
private engine;
|
|
4
|
-
private results;
|
|
5
3
|
private scenarioIds;
|
|
6
4
|
constructor();
|
|
7
5
|
private registerDefaultScenarios;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
avgTimeSavedRate: number;
|
|
13
|
-
avgTokenReductionRate: number;
|
|
14
|
-
avgFirstPassLift: number;
|
|
15
|
-
avgReworkReductionRate: number;
|
|
16
|
-
}>;
|
|
17
|
-
getExperimentalRoiSummary(): Promise<{
|
|
18
|
-
totalScenarios: number;
|
|
19
|
-
avgTimeSavedRate: number;
|
|
20
|
-
avgTokenReductionRate: number;
|
|
21
|
-
avgFirstPassLift: number;
|
|
22
|
-
avgReworkReductionRate: number;
|
|
23
|
-
conclusion: string;
|
|
6
|
+
getScenarioResults(): Promise<{
|
|
7
|
+
results: never[];
|
|
8
|
+
isRealData: boolean;
|
|
9
|
+
message: string;
|
|
24
10
|
}>;
|
|
25
11
|
}
|
|
26
12
|
//# sourceMappingURL=benchmark.service.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.service.d.ts","sourceRoot":"","sources":["../../src/benchmark/benchmark.service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"benchmark.service.d.ts","sourceRoot":"","sources":["../../src/benchmark/benchmark.service.ts"],"names":[],"mappings":"AAIA,qBACa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,WAAW,CAAgB;;IAOnC,OAAO,CAAC,wBAAwB;IA6B1B,kBAAkB;;;;;CAOzB"}
|
|
@@ -11,7 +11,6 @@ import { Injectable } from '@nestjs/common';
|
|
|
11
11
|
import { BenchmarkEngine } from '@devflow-tools/benchmark';
|
|
12
12
|
let BenchmarkService = class BenchmarkService {
|
|
13
13
|
constructor() {
|
|
14
|
-
this.results = [];
|
|
15
14
|
this.scenarioIds = [];
|
|
16
15
|
this.engine = new BenchmarkEngine();
|
|
17
16
|
this.registerDefaultScenarios();
|
|
@@ -43,53 +42,11 @@ let BenchmarkService = class BenchmarkService {
|
|
|
43
42
|
this.scenarioIds.push(s.id);
|
|
44
43
|
}
|
|
45
44
|
}
|
|
46
|
-
async runScenarioSet(scenarioIds) {
|
|
47
|
-
// Simulated runners — first version uses simulated data, ready for real execution integration
|
|
48
|
-
const baselineRunner = async (_prompt) => ({
|
|
49
|
-
prepTimeMs: 1000 + Math.random() * 500,
|
|
50
|
-
tokenCount: 500 + Math.floor(Math.random() * 200),
|
|
51
|
-
firstPassSuccess: Math.random() > 0.5,
|
|
52
|
-
reworkCount: Math.floor(Math.random() * 3),
|
|
53
|
-
durationMs: 5000 + Math.random() * 3000,
|
|
54
|
-
});
|
|
55
|
-
const devflowRunner = async (_prompt) => ({
|
|
56
|
-
prepTimeMs: 500 + Math.random() * 300,
|
|
57
|
-
tokenCount: 300 + Math.floor(Math.random() * 100),
|
|
58
|
-
firstPassSuccess: Math.random() > 0.2,
|
|
59
|
-
reworkCount: Math.floor(Math.random() * 2),
|
|
60
|
-
durationMs: 3000 + Math.random() * 2000,
|
|
61
|
-
});
|
|
62
|
-
const ids = scenarioIds ?? this.scenarioIds;
|
|
63
|
-
const results = await this.engine.runScenarioBatch(ids, baselineRunner, devflowRunner);
|
|
64
|
-
this.results = results;
|
|
65
|
-
return results;
|
|
66
|
-
}
|
|
67
45
|
async getScenarioResults() {
|
|
68
|
-
if (this.results.length === 0) {
|
|
69
|
-
await this.runScenarioSet();
|
|
70
|
-
}
|
|
71
|
-
return this.results;
|
|
72
|
-
}
|
|
73
|
-
async getComparisonSummary() {
|
|
74
|
-
const results = await this.getScenarioResults();
|
|
75
|
-
const n = results.length;
|
|
76
|
-
return {
|
|
77
|
-
totalScenarios: n,
|
|
78
|
-
avgTimeSavedRate: results.reduce((sum, r) => sum + r.comparison.timeSavedRate, 0) / n,
|
|
79
|
-
avgTokenReductionRate: results.reduce((sum, r) => sum + r.comparison.tokenReductionRate, 0) / n,
|
|
80
|
-
avgFirstPassLift: results.reduce((sum, r) => sum + r.comparison.firstPassLift, 0) / n,
|
|
81
|
-
avgReworkReductionRate: results.reduce((sum, r) => sum + r.comparison.reworkReductionRate, 0) / n,
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
async getExperimentalRoiSummary() {
|
|
85
|
-
const summary = await this.getComparisonSummary();
|
|
86
46
|
return {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
avgFirstPassLift: summary.avgFirstPassLift,
|
|
91
|
-
avgReworkReductionRate: summary.avgReworkReductionRate,
|
|
92
|
-
conclusion: 'DevFlow shows significant improvement in all metrics',
|
|
47
|
+
results: [],
|
|
48
|
+
isRealData: false,
|
|
49
|
+
message: 'No benchmark data yet. Real benchmarks require LLM runners — simulations are disabled to avoid misleading results.',
|
|
93
50
|
};
|
|
94
51
|
}
|
|
95
52
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.service.js","sourceRoot":"","sources":["../../src/benchmark/benchmark.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAIpD,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;
|
|
1
|
+
{"version":3,"file":"benchmark.service.js","sourceRoot":"","sources":["../../src/benchmark/benchmark.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAIpD,IAAM,gBAAgB,GAAtB,MAAM,gBAAgB;IAI3B;QAFQ,gBAAW,GAAa,EAAE,CAAC;QAGjC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAClC,CAAC;IAEO,wBAAwB;QAC9B,MAAM,SAAS,GAAwB;YACrC;gBACE,EAAE,EAAE,qBAAqB;gBACzB,QAAQ,EAAE,YAAY;gBACtB,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,CAAC;aACjB;YACD;gBACE,EAAE,EAAE,kBAAkB;gBACtB,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,aAAa;gBACnB,aAAa,EAAE,CAAC;gBAChB,kBAAkB,EAAE,CAAC;aACtB;YACD;gBACE,EAAE,EAAE,aAAa;gBACjB,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,MAAM;gBACZ,cAAc,EAAE,CAAC;aAClB;SACF,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,OAAO;YACL,OAAO,EAAE,EAAE;YACX,UAAU,EAAE,KAAK;YACjB,OAAO,EAAE,oHAAoH;SAC9H,CAAC;IACJ,CAAC;CACF,CAAA;AA7CY,gBAAgB;IAD5B,UAAU,EAAE;;GACA,gBAAgB,CA6C5B"}
|
|
@@ -12,10 +12,10 @@ export declare class WorkflowController {
|
|
|
12
12
|
listRuns(limit?: string, offset?: string): Promise<{
|
|
13
13
|
runId: string;
|
|
14
14
|
workflowName: string;
|
|
15
|
-
status: "
|
|
15
|
+
status: "completed" | "failed" | "running" | "canceled";
|
|
16
16
|
steps: {
|
|
17
17
|
stepId: string;
|
|
18
|
-
status: "
|
|
18
|
+
status: "failed" | "running" | "success" | "pending" | "skipped";
|
|
19
19
|
output: unknown;
|
|
20
20
|
error: string | undefined;
|
|
21
21
|
startedAt: number;
|
|
@@ -26,10 +26,10 @@ export declare class WorkflowController {
|
|
|
26
26
|
getRun(id: string): Promise<{
|
|
27
27
|
runId: string;
|
|
28
28
|
workflowName: string;
|
|
29
|
-
status: "
|
|
29
|
+
status: "completed" | "failed" | "running" | "canceled";
|
|
30
30
|
steps: {
|
|
31
31
|
stepId: string;
|
|
32
|
-
status: "
|
|
32
|
+
status: "failed" | "running" | "success" | "pending" | "skipped";
|
|
33
33
|
output: unknown;
|
|
34
34
|
error: string | undefined;
|
|
35
35
|
}[];
|
|
@@ -24,10 +24,10 @@ export declare class WorkflowService implements OnModuleInit {
|
|
|
24
24
|
listRuns(limit: number, offset: number): Promise<{
|
|
25
25
|
runId: string;
|
|
26
26
|
workflowName: string;
|
|
27
|
-
status: "
|
|
27
|
+
status: "completed" | "failed" | "running" | "canceled";
|
|
28
28
|
steps: {
|
|
29
29
|
stepId: string;
|
|
30
|
-
status: "
|
|
30
|
+
status: "failed" | "running" | "success" | "pending" | "skipped";
|
|
31
31
|
output: unknown;
|
|
32
32
|
error: string | undefined;
|
|
33
33
|
startedAt: number;
|
|
@@ -38,10 +38,10 @@ export declare class WorkflowService implements OnModuleInit {
|
|
|
38
38
|
getRun(runId: string): Promise<{
|
|
39
39
|
runId: string;
|
|
40
40
|
workflowName: string;
|
|
41
|
-
status: "
|
|
41
|
+
status: "completed" | "failed" | "running" | "canceled";
|
|
42
42
|
steps: {
|
|
43
43
|
stepId: string;
|
|
44
|
-
status: "
|
|
44
|
+
status: "failed" | "running" | "success" | "pending" | "skipped";
|
|
45
45
|
output: unknown;
|
|
46
46
|
error: string | undefined;
|
|
47
47
|
}[];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devflow-tools/server",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/main.js",
|
|
6
6
|
"scripts": {
|
|
@@ -12,15 +12,15 @@
|
|
|
12
12
|
"clean": "rm -rf dist"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@devflow-tools/adapters": "^0.12.
|
|
16
|
-
"@devflow-tools/benchmark": "^0.12.
|
|
17
|
-
"@devflow-tools/context-engine": "^0.12.
|
|
18
|
-
"@devflow-tools/database": "^0.12.
|
|
19
|
-
"@devflow-tools/knowledge-engine": "^0.12.
|
|
20
|
-
"@devflow-tools/memory-engine": "^0.12.
|
|
21
|
-
"@devflow-tools/sdk": "^0.12.
|
|
22
|
-
"@devflow-tools/telemetry": "^0.12.
|
|
23
|
-
"@devflow-tools/workflow-engine": "^0.12.
|
|
15
|
+
"@devflow-tools/adapters": "^0.12.5",
|
|
16
|
+
"@devflow-tools/benchmark": "^0.12.5",
|
|
17
|
+
"@devflow-tools/context-engine": "^0.12.5",
|
|
18
|
+
"@devflow-tools/database": "^0.12.5",
|
|
19
|
+
"@devflow-tools/knowledge-engine": "^0.12.5",
|
|
20
|
+
"@devflow-tools/memory-engine": "^0.12.5",
|
|
21
|
+
"@devflow-tools/sdk": "^0.12.5",
|
|
22
|
+
"@devflow-tools/telemetry": "^0.12.5",
|
|
23
|
+
"@devflow-tools/workflow-engine": "^0.12.5",
|
|
24
24
|
"@fastify/static": "^7.0.4",
|
|
25
25
|
"@nestjs/common": "^10.4.0",
|
|
26
26
|
"@nestjs/core": "^10.4.0",
|
|
@@ -39,5 +39,5 @@
|
|
|
39
39
|
"files": [
|
|
40
40
|
"dist"
|
|
41
41
|
],
|
|
42
|
-
"gitHead": "
|
|
42
|
+
"gitHead": "e8440cff84139ca0cc1786ee39be3b453974e000"
|
|
43
43
|
}
|