@tradejs/infra 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/ai.d.mts +36 -0
- package/dist/ai.d.ts +36 -0
- package/dist/ai.js +456 -0
- package/dist/ai.mjs +386 -0
- package/dist/chunk-EFARW5QE.mjs +297 -0
- package/dist/chunk-KVEZORMS.mjs +142 -0
- package/dist/ml.d.mts +2 -20
- package/dist/ml.d.ts +2 -20
- package/dist/ml.js +17 -0
- package/dist/ml.mjs +20 -124
- package/dist/mlDatasetFile-sRGWgR_o.d.mts +24 -0
- package/dist/mlDatasetFile-sRGWgR_o.d.ts +24 -0
- package/dist/redis.mjs +9 -287
- package/dist/userSettings.d.mts +29 -0
- package/dist/userSettings.d.ts +29 -0
- package/dist/userSettings.js +304 -0
- package/dist/userSettings.mjs +48 -0
- package/package.json +21 -3
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// src/mlDatasetFile.ts
|
|
2
|
+
import { once } from "events";
|
|
3
|
+
import { createReadStream, createWriteStream } from "fs";
|
|
4
|
+
import fs from "fs/promises";
|
|
5
|
+
import path from "path";
|
|
6
|
+
var DEFAULT_DIR = "data/ml/export";
|
|
7
|
+
var ML_DATASET_WRITE_BATCH_SIZE = 200;
|
|
8
|
+
var ML_CHUNK_FILE_RE = /^ml-dataset-(.+)-chunk-[^.]+\.jsonl$/;
|
|
9
|
+
var writerByPath = /* @__PURE__ */ new Map();
|
|
10
|
+
var toFileToken = (value) => value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "") || "any";
|
|
11
|
+
var getMlChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => path.join(
|
|
12
|
+
outDir,
|
|
13
|
+
`ml-dataset-${toFileToken(strategyName)}-chunk-${toFileToken(chunkId)}.jsonl`
|
|
14
|
+
);
|
|
15
|
+
var appendMlDatasetRow = async (params) => {
|
|
16
|
+
const { strategyName, chunkId, row, outDir = DEFAULT_DIR } = params;
|
|
17
|
+
const filePath = getMlChunkFilePath(strategyName, chunkId, outDir);
|
|
18
|
+
let state = writerByPath.get(filePath);
|
|
19
|
+
if (!state) {
|
|
20
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
21
|
+
const stream = createWriteStream(filePath, {
|
|
22
|
+
encoding: "utf8",
|
|
23
|
+
flags: "a"
|
|
24
|
+
});
|
|
25
|
+
state = {
|
|
26
|
+
filePath,
|
|
27
|
+
stream,
|
|
28
|
+
buffer: [],
|
|
29
|
+
writeQueue: Promise.resolve(),
|
|
30
|
+
closed: false
|
|
31
|
+
};
|
|
32
|
+
writerByPath.set(filePath, state);
|
|
33
|
+
}
|
|
34
|
+
if (state.closed) {
|
|
35
|
+
throw new Error(`ML dataset writer is closed: ${filePath}`);
|
|
36
|
+
}
|
|
37
|
+
state.buffer.push(`${JSON.stringify(row)}
|
|
38
|
+
`);
|
|
39
|
+
if (state.buffer.length >= ML_DATASET_WRITE_BATCH_SIZE) {
|
|
40
|
+
await flushMlDatasetWriter(filePath);
|
|
41
|
+
}
|
|
42
|
+
return filePath;
|
|
43
|
+
};
|
|
44
|
+
var flushState = async (state) => {
|
|
45
|
+
if (state.closed || state.buffer.length === 0) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const chunk = state.buffer.join("");
|
|
49
|
+
state.buffer = [];
|
|
50
|
+
if (!state.stream.write(chunk)) {
|
|
51
|
+
await once(state.stream, "drain");
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
var flushMlDatasetWriter = async (filePath) => {
|
|
55
|
+
const state = writerByPath.get(filePath);
|
|
56
|
+
if (!state || state.closed) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
state.writeQueue = state.writeQueue.then(() => flushState(state));
|
|
60
|
+
await state.writeQueue;
|
|
61
|
+
};
|
|
62
|
+
var closeState = async (state) => {
|
|
63
|
+
if (state.closed) {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
await flushState(state);
|
|
67
|
+
state.closed = true;
|
|
68
|
+
state.stream.end();
|
|
69
|
+
await Promise.all([
|
|
70
|
+
once(state.stream, "finish"),
|
|
71
|
+
once(state.stream, "close")
|
|
72
|
+
]);
|
|
73
|
+
};
|
|
74
|
+
var closeMlDatasetWriter = async (filePath) => {
|
|
75
|
+
const state = writerByPath.get(filePath);
|
|
76
|
+
if (!state) return;
|
|
77
|
+
state.writeQueue = state.writeQueue.then(() => closeState(state));
|
|
78
|
+
await state.writeQueue;
|
|
79
|
+
writerByPath.delete(filePath);
|
|
80
|
+
};
|
|
81
|
+
var closeAllMlDatasetWriters = async () => {
|
|
82
|
+
const filePaths = [...writerByPath.keys()];
|
|
83
|
+
for (const filePath of filePaths) {
|
|
84
|
+
await closeMlDatasetWriter(filePath);
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
var listMlChunkFiles = async (params) => {
|
|
88
|
+
const { strategyName, outDir = DEFAULT_DIR } = params;
|
|
89
|
+
const prefix = `ml-dataset-${toFileToken(strategyName)}-chunk-`;
|
|
90
|
+
let entries = [];
|
|
91
|
+
try {
|
|
92
|
+
entries = await fs.readdir(outDir);
|
|
93
|
+
} catch (error) {
|
|
94
|
+
return [];
|
|
95
|
+
}
|
|
96
|
+
return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => path.join(outDir, name)).sort();
|
|
97
|
+
};
|
|
98
|
+
var listMlChunkStrategies = async (params) => {
|
|
99
|
+
const outDir = params?.outDir ?? DEFAULT_DIR;
|
|
100
|
+
let entries = [];
|
|
101
|
+
try {
|
|
102
|
+
entries = await fs.readdir(outDir);
|
|
103
|
+
} catch {
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
return [
|
|
107
|
+
...new Set(
|
|
108
|
+
entries.map((name) => name.match(ML_CHUNK_FILE_RE)?.[1] || "").filter(Boolean)
|
|
109
|
+
)
|
|
110
|
+
].sort();
|
|
111
|
+
};
|
|
112
|
+
var mergeJsonlFiles = async (params) => {
|
|
113
|
+
const { filePaths, outPath } = params;
|
|
114
|
+
await fs.mkdir(path.dirname(outPath), { recursive: true });
|
|
115
|
+
const stream = createWriteStream(outPath, { encoding: "utf8" });
|
|
116
|
+
const done = Promise.all([once(stream, "finish"), once(stream, "close")]);
|
|
117
|
+
try {
|
|
118
|
+
for (const filePath of filePaths) {
|
|
119
|
+
const reader = createReadStream(filePath, { encoding: "utf8" });
|
|
120
|
+
for await (const chunk of reader) {
|
|
121
|
+
if (!stream.write(chunk)) {
|
|
122
|
+
await once(stream, "drain");
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
} finally {
|
|
127
|
+
stream.end();
|
|
128
|
+
await done;
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
export {
|
|
133
|
+
toFileToken,
|
|
134
|
+
getMlChunkFilePath,
|
|
135
|
+
appendMlDatasetRow,
|
|
136
|
+
flushMlDatasetWriter,
|
|
137
|
+
closeMlDatasetWriter,
|
|
138
|
+
closeAllMlDatasetWriters,
|
|
139
|
+
listMlChunkFiles,
|
|
140
|
+
listMlChunkStrategies,
|
|
141
|
+
mergeJsonlFiles
|
|
142
|
+
};
|
package/dist/ml.d.mts
CHANGED
|
@@ -1,22 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
declare const getMlChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
|
|
3
|
-
declare const appendMlDatasetRow: (params: {
|
|
4
|
-
strategyName: string;
|
|
5
|
-
chunkId: string;
|
|
6
|
-
row: Record<string, number | string | null>;
|
|
7
|
-
outDir?: string;
|
|
8
|
-
}) => Promise<string>;
|
|
9
|
-
declare const flushMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
10
|
-
declare const closeMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
11
|
-
declare const closeAllMlDatasetWriters: () => Promise<void>;
|
|
12
|
-
declare const listMlChunkFiles: (params: {
|
|
13
|
-
strategyName: string;
|
|
14
|
-
outDir?: string;
|
|
15
|
-
}) => Promise<string[]>;
|
|
16
|
-
declare const mergeJsonlFiles: (params: {
|
|
17
|
-
filePaths: string[];
|
|
18
|
-
outPath: string;
|
|
19
|
-
}) => Promise<void>;
|
|
1
|
+
export { a as appendMlDatasetRow, c as closeAllMlDatasetWriters, b as closeMlDatasetWriter, f as flushMlDatasetWriter, g as getMlChunkFilePath, l as listMlChunkFiles, d as listMlChunkStrategies, m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-sRGWgR_o.mjs';
|
|
20
2
|
|
|
21
3
|
type MlPredictResponse = {
|
|
22
4
|
probability: number;
|
|
@@ -134,4 +116,4 @@ type MlSeriesAnalysisSummary = Record<string, number>;
|
|
|
134
116
|
declare const analyzeMlSeriesWindow: (input: MlSeriesAnalysisInput) => MlSeriesAnalysisSummary;
|
|
135
117
|
declare const buildMlSeriesAlignment: (left: MlSeriesAnalysisSummary | undefined, right: MlSeriesAnalysisSummary | undefined) => MlSeriesAnalysisSummary;
|
|
136
118
|
|
|
137
|
-
export { type LookaheadViolation, type MlPredictParams, type MlPredictResponse, type MlResultRecord, type MlSeriesAnalysisCandle, type MlSeriesAnalysisSummary, type MlSignalRecord, analyzeMlSeriesWindow,
|
|
119
|
+
export { type LookaheadViolation, type MlPredictParams, type MlPredictResponse, type MlResultRecord, type MlSeriesAnalysisCandle, type MlSeriesAnalysisSummary, type MlSignalRecord, analyzeMlSeriesWindow, buildMlFeatures, buildMlSeriesAlignment, buildMlTrainingRow, computeWindowBoundaries, fetchMlThreshold, findLookaheadViolations, isDerivedDatasetFileName, isTimestampFeatureKey, toIsoUtcOrNull, trimMlTrainingRowWindows };
|
package/dist/ml.d.ts
CHANGED
|
@@ -1,22 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
declare const getMlChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
|
|
3
|
-
declare const appendMlDatasetRow: (params: {
|
|
4
|
-
strategyName: string;
|
|
5
|
-
chunkId: string;
|
|
6
|
-
row: Record<string, number | string | null>;
|
|
7
|
-
outDir?: string;
|
|
8
|
-
}) => Promise<string>;
|
|
9
|
-
declare const flushMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
10
|
-
declare const closeMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
11
|
-
declare const closeAllMlDatasetWriters: () => Promise<void>;
|
|
12
|
-
declare const listMlChunkFiles: (params: {
|
|
13
|
-
strategyName: string;
|
|
14
|
-
outDir?: string;
|
|
15
|
-
}) => Promise<string[]>;
|
|
16
|
-
declare const mergeJsonlFiles: (params: {
|
|
17
|
-
filePaths: string[];
|
|
18
|
-
outPath: string;
|
|
19
|
-
}) => Promise<void>;
|
|
1
|
+
export { a as appendMlDatasetRow, c as closeAllMlDatasetWriters, b as closeMlDatasetWriter, f as flushMlDatasetWriter, g as getMlChunkFilePath, l as listMlChunkFiles, d as listMlChunkStrategies, m as mergeJsonlFiles, t as toFileToken } from './mlDatasetFile-sRGWgR_o.js';
|
|
20
2
|
|
|
21
3
|
type MlPredictResponse = {
|
|
22
4
|
probability: number;
|
|
@@ -134,4 +116,4 @@ type MlSeriesAnalysisSummary = Record<string, number>;
|
|
|
134
116
|
declare const analyzeMlSeriesWindow: (input: MlSeriesAnalysisInput) => MlSeriesAnalysisSummary;
|
|
135
117
|
declare const buildMlSeriesAlignment: (left: MlSeriesAnalysisSummary | undefined, right: MlSeriesAnalysisSummary | undefined) => MlSeriesAnalysisSummary;
|
|
136
118
|
|
|
137
|
-
export { type LookaheadViolation, type MlPredictParams, type MlPredictResponse, type MlResultRecord, type MlSeriesAnalysisCandle, type MlSeriesAnalysisSummary, type MlSignalRecord, analyzeMlSeriesWindow,
|
|
119
|
+
export { type LookaheadViolation, type MlPredictParams, type MlPredictResponse, type MlResultRecord, type MlSeriesAnalysisCandle, type MlSeriesAnalysisSummary, type MlSignalRecord, analyzeMlSeriesWindow, buildMlFeatures, buildMlSeriesAlignment, buildMlTrainingRow, computeWindowBoundaries, fetchMlThreshold, findLookaheadViolations, isDerivedDatasetFileName, isTimestampFeatureKey, toIsoUtcOrNull, trimMlTrainingRowWindows };
|
package/dist/ml.js
CHANGED
|
@@ -45,6 +45,7 @@ __export(ml_exports, {
|
|
|
45
45
|
isDerivedDatasetFileName: () => isDerivedDatasetFileName,
|
|
46
46
|
isTimestampFeatureKey: () => isTimestampFeatureKey,
|
|
47
47
|
listMlChunkFiles: () => listMlChunkFiles,
|
|
48
|
+
listMlChunkStrategies: () => listMlChunkStrategies,
|
|
48
49
|
mergeJsonlFiles: () => mergeJsonlFiles,
|
|
49
50
|
toFileToken: () => toFileToken,
|
|
50
51
|
toIsoUtcOrNull: () => toIsoUtcOrNull,
|
|
@@ -59,6 +60,7 @@ var import_promises = __toESM(require("fs/promises"));
|
|
|
59
60
|
var import_path = __toESM(require("path"));
|
|
60
61
|
var DEFAULT_DIR = "data/ml/export";
|
|
61
62
|
var ML_DATASET_WRITE_BATCH_SIZE = 200;
|
|
63
|
+
var ML_CHUNK_FILE_RE = /^ml-dataset-(.+)-chunk-[^.]+\.jsonl$/;
|
|
62
64
|
var writerByPath = /* @__PURE__ */ new Map();
|
|
63
65
|
var toFileToken = (value) => value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "") || "any";
|
|
64
66
|
var getMlChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => import_path.default.join(
|
|
@@ -148,6 +150,20 @@ var listMlChunkFiles = async (params) => {
|
|
|
148
150
|
}
|
|
149
151
|
return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => import_path.default.join(outDir, name)).sort();
|
|
150
152
|
};
|
|
153
|
+
var listMlChunkStrategies = async (params) => {
|
|
154
|
+
const outDir = params?.outDir ?? DEFAULT_DIR;
|
|
155
|
+
let entries = [];
|
|
156
|
+
try {
|
|
157
|
+
entries = await import_promises.default.readdir(outDir);
|
|
158
|
+
} catch {
|
|
159
|
+
return [];
|
|
160
|
+
}
|
|
161
|
+
return [
|
|
162
|
+
...new Set(
|
|
163
|
+
entries.map((name) => name.match(ML_CHUNK_FILE_RE)?.[1] || "").filter(Boolean)
|
|
164
|
+
)
|
|
165
|
+
].sort();
|
|
166
|
+
};
|
|
151
167
|
var mergeJsonlFiles = async (params) => {
|
|
152
168
|
const { filePaths, outPath } = params;
|
|
153
169
|
await import_promises.default.mkdir(import_path.default.dirname(outPath), { recursive: true });
|
|
@@ -1605,6 +1621,7 @@ var findLookaheadViolations = (row) => {
|
|
|
1605
1621
|
isDerivedDatasetFileName,
|
|
1606
1622
|
isTimestampFeatureKey,
|
|
1607
1623
|
listMlChunkFiles,
|
|
1624
|
+
listMlChunkStrategies,
|
|
1608
1625
|
mergeJsonlFiles,
|
|
1609
1626
|
toFileToken,
|
|
1610
1627
|
toIsoUtcOrNull,
|
package/dist/ml.mjs
CHANGED
|
@@ -1,142 +1,37 @@
|
|
|
1
|
+
import {
|
|
2
|
+
appendMlDatasetRow,
|
|
3
|
+
closeAllMlDatasetWriters,
|
|
4
|
+
closeMlDatasetWriter,
|
|
5
|
+
flushMlDatasetWriter,
|
|
6
|
+
getMlChunkFilePath,
|
|
7
|
+
listMlChunkFiles,
|
|
8
|
+
listMlChunkStrategies,
|
|
9
|
+
mergeJsonlFiles,
|
|
10
|
+
toFileToken
|
|
11
|
+
} from "./chunk-KVEZORMS.mjs";
|
|
1
12
|
import {
|
|
2
13
|
logger
|
|
3
14
|
} from "./chunk-LNFUOXDW.mjs";
|
|
4
15
|
|
|
5
|
-
// src/mlDatasetFile.ts
|
|
6
|
-
import { once } from "events";
|
|
7
|
-
import { createReadStream, createWriteStream } from "fs";
|
|
8
|
-
import fs from "fs/promises";
|
|
9
|
-
import path from "path";
|
|
10
|
-
var DEFAULT_DIR = "data/ml/export";
|
|
11
|
-
var ML_DATASET_WRITE_BATCH_SIZE = 200;
|
|
12
|
-
var writerByPath = /* @__PURE__ */ new Map();
|
|
13
|
-
var toFileToken = (value) => value.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "_").replace(/^_+|_+$/g, "") || "any";
|
|
14
|
-
var getMlChunkFilePath = (strategyName, chunkId, outDir = DEFAULT_DIR) => path.join(
|
|
15
|
-
outDir,
|
|
16
|
-
`ml-dataset-${toFileToken(strategyName)}-chunk-${toFileToken(chunkId)}.jsonl`
|
|
17
|
-
);
|
|
18
|
-
var appendMlDatasetRow = async (params) => {
|
|
19
|
-
const { strategyName, chunkId, row, outDir = DEFAULT_DIR } = params;
|
|
20
|
-
const filePath = getMlChunkFilePath(strategyName, chunkId, outDir);
|
|
21
|
-
let state = writerByPath.get(filePath);
|
|
22
|
-
if (!state) {
|
|
23
|
-
await fs.mkdir(outDir, { recursive: true });
|
|
24
|
-
const stream = createWriteStream(filePath, {
|
|
25
|
-
encoding: "utf8",
|
|
26
|
-
flags: "a"
|
|
27
|
-
});
|
|
28
|
-
state = {
|
|
29
|
-
filePath,
|
|
30
|
-
stream,
|
|
31
|
-
buffer: [],
|
|
32
|
-
writeQueue: Promise.resolve(),
|
|
33
|
-
closed: false
|
|
34
|
-
};
|
|
35
|
-
writerByPath.set(filePath, state);
|
|
36
|
-
}
|
|
37
|
-
if (state.closed) {
|
|
38
|
-
throw new Error(`ML dataset writer is closed: ${filePath}`);
|
|
39
|
-
}
|
|
40
|
-
state.buffer.push(`${JSON.stringify(row)}
|
|
41
|
-
`);
|
|
42
|
-
if (state.buffer.length >= ML_DATASET_WRITE_BATCH_SIZE) {
|
|
43
|
-
await flushMlDatasetWriter(filePath);
|
|
44
|
-
}
|
|
45
|
-
return filePath;
|
|
46
|
-
};
|
|
47
|
-
var flushState = async (state) => {
|
|
48
|
-
if (state.closed || state.buffer.length === 0) {
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
const chunk = state.buffer.join("");
|
|
52
|
-
state.buffer = [];
|
|
53
|
-
if (!state.stream.write(chunk)) {
|
|
54
|
-
await once(state.stream, "drain");
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
var flushMlDatasetWriter = async (filePath) => {
|
|
58
|
-
const state = writerByPath.get(filePath);
|
|
59
|
-
if (!state || state.closed) {
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
state.writeQueue = state.writeQueue.then(() => flushState(state));
|
|
63
|
-
await state.writeQueue;
|
|
64
|
-
};
|
|
65
|
-
var closeState = async (state) => {
|
|
66
|
-
if (state.closed) {
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
await flushState(state);
|
|
70
|
-
state.closed = true;
|
|
71
|
-
state.stream.end();
|
|
72
|
-
await Promise.all([
|
|
73
|
-
once(state.stream, "finish"),
|
|
74
|
-
once(state.stream, "close")
|
|
75
|
-
]);
|
|
76
|
-
};
|
|
77
|
-
var closeMlDatasetWriter = async (filePath) => {
|
|
78
|
-
const state = writerByPath.get(filePath);
|
|
79
|
-
if (!state) return;
|
|
80
|
-
state.writeQueue = state.writeQueue.then(() => closeState(state));
|
|
81
|
-
await state.writeQueue;
|
|
82
|
-
writerByPath.delete(filePath);
|
|
83
|
-
};
|
|
84
|
-
var closeAllMlDatasetWriters = async () => {
|
|
85
|
-
const filePaths = [...writerByPath.keys()];
|
|
86
|
-
for (const filePath of filePaths) {
|
|
87
|
-
await closeMlDatasetWriter(filePath);
|
|
88
|
-
}
|
|
89
|
-
};
|
|
90
|
-
var listMlChunkFiles = async (params) => {
|
|
91
|
-
const { strategyName, outDir = DEFAULT_DIR } = params;
|
|
92
|
-
const prefix = `ml-dataset-${toFileToken(strategyName)}-chunk-`;
|
|
93
|
-
let entries = [];
|
|
94
|
-
try {
|
|
95
|
-
entries = await fs.readdir(outDir);
|
|
96
|
-
} catch (error) {
|
|
97
|
-
return [];
|
|
98
|
-
}
|
|
99
|
-
return entries.filter((name) => name.startsWith(prefix) && name.endsWith(".jsonl")).map((name) => path.join(outDir, name)).sort();
|
|
100
|
-
};
|
|
101
|
-
var mergeJsonlFiles = async (params) => {
|
|
102
|
-
const { filePaths, outPath } = params;
|
|
103
|
-
await fs.mkdir(path.dirname(outPath), { recursive: true });
|
|
104
|
-
const stream = createWriteStream(outPath, { encoding: "utf8" });
|
|
105
|
-
const done = Promise.all([once(stream, "finish"), once(stream, "close")]);
|
|
106
|
-
try {
|
|
107
|
-
for (const filePath of filePaths) {
|
|
108
|
-
const reader = createReadStream(filePath, { encoding: "utf8" });
|
|
109
|
-
for await (const chunk of reader) {
|
|
110
|
-
if (!stream.write(chunk)) {
|
|
111
|
-
await once(stream, "drain");
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
} finally {
|
|
116
|
-
stream.end();
|
|
117
|
-
await done;
|
|
118
|
-
}
|
|
119
|
-
};
|
|
120
|
-
|
|
121
16
|
// src/mlGrpc.ts
|
|
122
|
-
import
|
|
17
|
+
import path from "path";
|
|
123
18
|
import * as grpc from "@grpc/grpc-js";
|
|
124
19
|
import * as protoLoader from "@grpc/proto-loader";
|
|
125
|
-
import
|
|
20
|
+
import fs from "fs";
|
|
126
21
|
var clientCache = /* @__PURE__ */ new Map();
|
|
127
22
|
var resolveProtoPath = (projectRoot, protoPath) => {
|
|
128
|
-
if (protoPath &&
|
|
23
|
+
if (protoPath && fs.existsSync(protoPath)) {
|
|
129
24
|
return protoPath;
|
|
130
25
|
}
|
|
131
26
|
const explicitRoot = String(projectRoot || "").trim();
|
|
132
|
-
const root = explicitRoot ?
|
|
27
|
+
const root = explicitRoot ? path.resolve(explicitRoot) : String(process.env.PROJECT_CWD || "").trim() ? path.resolve(String(process.env.PROJECT_CWD || "").trim()) : process.cwd();
|
|
133
28
|
const candidates = [
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
29
|
+
path.resolve(__dirname, "../proto/ml_infer.proto"),
|
|
30
|
+
path.resolve(__dirname, "../../proto/ml_infer.proto"),
|
|
31
|
+
path.resolve(root, "proto/ml_infer.proto")
|
|
137
32
|
];
|
|
138
33
|
for (const candidate of candidates) {
|
|
139
|
-
if (
|
|
34
|
+
if (fs.existsSync(candidate)) {
|
|
140
35
|
return candidate;
|
|
141
36
|
}
|
|
142
37
|
}
|
|
@@ -1513,6 +1408,7 @@ export {
|
|
|
1513
1408
|
isDerivedDatasetFileName,
|
|
1514
1409
|
isTimestampFeatureKey,
|
|
1515
1410
|
listMlChunkFiles,
|
|
1411
|
+
listMlChunkStrategies,
|
|
1516
1412
|
mergeJsonlFiles,
|
|
1517
1413
|
toFileToken,
|
|
1518
1414
|
toIsoUtcOrNull,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
declare const toFileToken: (value: string) => string;
|
|
2
|
+
declare const getMlChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
|
|
3
|
+
declare const appendMlDatasetRow: (params: {
|
|
4
|
+
strategyName: string;
|
|
5
|
+
chunkId: string;
|
|
6
|
+
row: Record<string, number | string | null>;
|
|
7
|
+
outDir?: string;
|
|
8
|
+
}) => Promise<string>;
|
|
9
|
+
declare const flushMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
10
|
+
declare const closeMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
11
|
+
declare const closeAllMlDatasetWriters: () => Promise<void>;
|
|
12
|
+
declare const listMlChunkFiles: (params: {
|
|
13
|
+
strategyName: string;
|
|
14
|
+
outDir?: string;
|
|
15
|
+
}) => Promise<string[]>;
|
|
16
|
+
declare const listMlChunkStrategies: (params?: {
|
|
17
|
+
outDir?: string;
|
|
18
|
+
}) => Promise<string[]>;
|
|
19
|
+
declare const mergeJsonlFiles: (params: {
|
|
20
|
+
filePaths: string[];
|
|
21
|
+
outPath: string;
|
|
22
|
+
}) => Promise<void>;
|
|
23
|
+
|
|
24
|
+
export { appendMlDatasetRow as a, closeMlDatasetWriter as b, closeAllMlDatasetWriters as c, listMlChunkStrategies as d, flushMlDatasetWriter as f, getMlChunkFilePath as g, listMlChunkFiles as l, mergeJsonlFiles as m, toFileToken as t };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
declare const toFileToken: (value: string) => string;
|
|
2
|
+
declare const getMlChunkFilePath: (strategyName: string, chunkId: string, outDir?: string) => string;
|
|
3
|
+
declare const appendMlDatasetRow: (params: {
|
|
4
|
+
strategyName: string;
|
|
5
|
+
chunkId: string;
|
|
6
|
+
row: Record<string, number | string | null>;
|
|
7
|
+
outDir?: string;
|
|
8
|
+
}) => Promise<string>;
|
|
9
|
+
declare const flushMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
10
|
+
declare const closeMlDatasetWriter: (filePath: string) => Promise<void>;
|
|
11
|
+
declare const closeAllMlDatasetWriters: () => Promise<void>;
|
|
12
|
+
declare const listMlChunkFiles: (params: {
|
|
13
|
+
strategyName: string;
|
|
14
|
+
outDir?: string;
|
|
15
|
+
}) => Promise<string[]>;
|
|
16
|
+
declare const listMlChunkStrategies: (params?: {
|
|
17
|
+
outDir?: string;
|
|
18
|
+
}) => Promise<string[]>;
|
|
19
|
+
declare const mergeJsonlFiles: (params: {
|
|
20
|
+
filePaths: string[];
|
|
21
|
+
outPath: string;
|
|
22
|
+
}) => Promise<void>;
|
|
23
|
+
|
|
24
|
+
export { appendMlDatasetRow as a, closeMlDatasetWriter as b, closeAllMlDatasetWriters as c, listMlChunkStrategies as d, flushMlDatasetWriter as f, getMlChunkFilePath as g, listMlChunkFiles as l, mergeJsonlFiles as m, toFileToken as t };
|