@tyvm/knowhow 0.0.101 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/agents/base/base.ts +34 -6
- package/src/chat/modules/AgentModule.ts +24 -10
- package/src/chat/modules/InternalChatModule.ts +6 -0
- package/src/chat/modules/RemoteSyncModule.ts +447 -0
- package/src/chat/types.ts +2 -0
- package/src/cloudWorker.ts +48 -30
- package/src/config.ts +2 -0
- package/src/fileSync.ts +153 -9
- package/src/hashes.ts +52 -0
- package/src/services/AgentSyncFs.ts +24 -2
- package/src/services/AgentSyncKnowhowWeb.ts +27 -5
- package/src/services/KnowhowClient.ts +69 -3
- package/src/services/S3.ts +15 -5
- package/src/services/SessionManager.ts +2 -0
- package/src/services/script-execution/ScriptPolicy.ts +0 -44
- package/src/types.ts +3 -0
- package/src/worker.ts +70 -4
- package/ts_build/package.json +2 -2
- package/ts_build/src/agents/base/base.d.ts +1 -0
- package/ts_build/src/agents/base/base.js +23 -4
- package/ts_build/src/agents/base/base.js.map +1 -1
- package/ts_build/src/chat/modules/AgentModule.d.ts +2 -1
- package/ts_build/src/chat/modules/AgentModule.js +12 -7
- package/ts_build/src/chat/modules/AgentModule.js.map +1 -1
- package/ts_build/src/chat/modules/InternalChatModule.d.ts +1 -0
- package/ts_build/src/chat/modules/InternalChatModule.js +6 -0
- package/ts_build/src/chat/modules/InternalChatModule.js.map +1 -1
- package/ts_build/src/chat/modules/RemoteSyncModule.d.ts +27 -0
- package/ts_build/src/chat/modules/RemoteSyncModule.js +282 -0
- package/ts_build/src/chat/modules/RemoteSyncModule.js.map +1 -0
- package/ts_build/src/chat/types.d.ts +2 -0
- package/ts_build/src/cloudWorker.js +38 -25
- package/ts_build/src/cloudWorker.js.map +1 -1
- package/ts_build/src/config.js +1 -0
- package/ts_build/src/config.js.map +1 -1
- package/ts_build/src/fileSync.d.ts +3 -0
- package/ts_build/src/fileSync.js +104 -6
- package/ts_build/src/fileSync.js.map +1 -1
- package/ts_build/src/hashes.d.ts +4 -0
- package/ts_build/src/hashes.js +34 -0
- package/ts_build/src/hashes.js.map +1 -1
- package/ts_build/src/services/AgentSyncFs.d.ts +1 -0
- package/ts_build/src/services/AgentSyncFs.js +13 -2
- package/ts_build/src/services/AgentSyncFs.js.map +1 -1
- package/ts_build/src/services/AgentSyncKnowhowWeb.d.ts +1 -0
- package/ts_build/src/services/AgentSyncKnowhowWeb.js +13 -2
- package/ts_build/src/services/AgentSyncKnowhowWeb.js.map +1 -1
- package/ts_build/src/services/KnowhowClient.d.ts +25 -1
- package/ts_build/src/services/KnowhowClient.js +14 -1
- package/ts_build/src/services/KnowhowClient.js.map +1 -1
- package/ts_build/src/services/Mcp.d.ts +219 -406
- package/ts_build/src/services/S3.js +14 -4
- package/ts_build/src/services/S3.js.map +1 -1
- package/ts_build/src/services/SessionManager.js +2 -0
- package/ts_build/src/services/SessionManager.js.map +1 -1
- package/ts_build/src/services/script-execution/ScriptPolicy.js +0 -35
- package/ts_build/src/services/script-execution/ScriptPolicy.js.map +1 -1
- package/ts_build/src/types.d.ts +2 -0
- package/ts_build/src/types.js.map +1 -1
- package/ts_build/src/worker.js +51 -2
- package/ts_build/src/worker.js.map +1 -1
package/src/chat/types.ts
CHANGED
|
@@ -84,6 +84,7 @@ export interface TaskInfo {
|
|
|
84
84
|
taskId: string;
|
|
85
85
|
knowhowMessageId?: string;
|
|
86
86
|
knowhowTaskId?: string;
|
|
87
|
+
chatSessionId?: string;
|
|
87
88
|
agentName: string;
|
|
88
89
|
agent: BaseAgent;
|
|
89
90
|
initialInput: string;
|
|
@@ -98,6 +99,7 @@ export interface TaskInfo {
|
|
|
98
99
|
export interface ChatSession {
|
|
99
100
|
knowhowMessageId?: string;
|
|
100
101
|
knowhowTaskId?: string;
|
|
102
|
+
chatSessionId?: string;
|
|
101
103
|
sessionId: string;
|
|
102
104
|
taskId: string;
|
|
103
105
|
agentName: string;
|
package/src/cloudWorker.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import * as fs from "fs";
|
|
2
2
|
import * as path from "path";
|
|
3
|
-
import { glob } from "glob";
|
|
4
3
|
import { KnowhowSimpleClient, KNOWHOW_API_URL } from "./services/KnowhowClient";
|
|
5
4
|
import { loadJwt } from "./login";
|
|
6
5
|
import { getConfig, updateConfig, getLanguageConfig } from "./config";
|
|
@@ -23,6 +22,26 @@ interface FileToSync {
|
|
|
23
22
|
localPath: string;
|
|
24
23
|
remotePath: string;
|
|
25
24
|
downloadLocalPath?: string; // override localPath used when worker downloads the file
|
|
25
|
+
isDirectory?: boolean; // true if this represents a whole directory
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Recursively list all files in a local directory, returning relative paths
|
|
30
|
+
*/
|
|
31
|
+
function listFilesRecursively(dir: string): string[] {
|
|
32
|
+
const results: string[] = [];
|
|
33
|
+
if (!fs.existsSync(dir)) return results;
|
|
34
|
+
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
35
|
+
for (const entry of entries) {
|
|
36
|
+
if (entry.isDirectory()) {
|
|
37
|
+
listFilesRecursively(path.join(dir, entry.name)).forEach((f) =>
|
|
38
|
+
results.push(entry.name + "/" + f)
|
|
39
|
+
);
|
|
40
|
+
} else {
|
|
41
|
+
results.push(entry.name);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return results;
|
|
26
45
|
}
|
|
27
46
|
|
|
28
47
|
/**
|
|
@@ -48,6 +67,8 @@ function buildWorkerConfigJson(config: Config, files: { remotePath: string; loca
|
|
|
48
67
|
|
|
49
68
|
/**
|
|
50
69
|
* Collect all files from the .knowhow directory that should be synced
|
|
70
|
+
* Uses directory-level entries where possible so the worker config stays compact
|
|
71
|
+
* and the folder upload/download feature handles individual files automatically.
|
|
51
72
|
*/
|
|
52
73
|
async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
|
|
53
74
|
const filesToSync: FileToSync[] = [];
|
|
@@ -59,39 +80,24 @@ async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
|
|
|
59
80
|
}
|
|
60
81
|
};
|
|
61
82
|
|
|
83
|
+
// Helper to add a directory entry if it exists (trailing slash = directory mode)
|
|
84
|
+
const addDirIfExists = (localPath: string, remotePath: string) => {
|
|
85
|
+
if (fs.existsSync(localPath)) {
|
|
86
|
+
filesToSync.push({ localPath: localPath + "/", remotePath: remotePath + "/", isDirectory: true });
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
|
|
62
90
|
// .knowhow/language.json
|
|
63
91
|
addIfExists(".knowhow/language.json", `${projectName}/.knowhow/language.json`);
|
|
64
92
|
|
|
65
93
|
// .knowhow/hashes.json
|
|
66
94
|
addIfExists(".knowhow/hashes.json", `${projectName}/.knowhow/hashes.json`);
|
|
67
95
|
|
|
68
|
-
//
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
filesToSync.push({ localPath: filePath, remotePath });
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
// .knowhow/scripts/**/* (if exists)
|
|
77
|
-
if (fs.existsSync(".knowhow/scripts")) {
|
|
78
|
-
const scriptFiles = await glob(".knowhow/scripts/**/*", { nodir: true });
|
|
79
|
-
for (const filePath of scriptFiles) {
|
|
80
|
-
const relativeToDotKnowhow = filePath.replace(/^\.knowhow\//, "");
|
|
81
|
-
const remotePath = `${projectName}/.knowhow/${relativeToDotKnowhow}`;
|
|
82
|
-
filesToSync.push({ localPath: filePath, remotePath });
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
// .knowhow/skills/**/* (if exists)
|
|
87
|
-
if (fs.existsSync(".knowhow/skills")) {
|
|
88
|
-
const skillFiles = await glob(".knowhow/skills/**/*", { nodir: true });
|
|
89
|
-
for (const filePath of skillFiles) {
|
|
90
|
-
const relativeToDotKnowhow = filePath.replace(/^\.knowhow\//, "");
|
|
91
|
-
const remotePath = `${projectName}/.knowhow/${relativeToDotKnowhow}`;
|
|
92
|
-
filesToSync.push({ localPath: filePath, remotePath });
|
|
93
|
-
}
|
|
94
|
-
}
|
|
96
|
+
// Directories — use trailing-slash entries so folder upload/download handles them
|
|
97
|
+
addDirIfExists(".knowhow/prompts", `${projectName}/.knowhow/prompts`);
|
|
98
|
+
addDirIfExists(".knowhow/scripts", `${projectName}/.knowhow/scripts`);
|
|
99
|
+
addDirIfExists(".knowhow/skills", `${projectName}/.knowhow/skills`);
|
|
100
|
+
addDirIfExists(".knowhow/tasks", `${projectName}/.knowhow/tasks`);
|
|
95
101
|
|
|
96
102
|
return filesToSync;
|
|
97
103
|
}
|
|
@@ -264,8 +270,20 @@ export async function cloudWorker(options: CloudWorkerOptions) {
|
|
|
264
270
|
|
|
265
271
|
for (const file of allFiles) {
|
|
266
272
|
try {
|
|
267
|
-
|
|
268
|
-
|
|
273
|
+
if (file.isDirectory) {
|
|
274
|
+
// Upload all files recursively in the local directory
|
|
275
|
+
const localDir = file.localPath.endsWith("/") ? file.localPath : file.localPath + "/";
|
|
276
|
+
const remoteDir = file.remotePath.endsWith("/") ? file.remotePath : file.remotePath + "/";
|
|
277
|
+
const relFiles = listFilesRecursively(localDir);
|
|
278
|
+
console.log(` 📁 Uploading directory ${localDir} → ${remoteDir} (${relFiles.length} files)`);
|
|
279
|
+
for (const relFile of relFiles) {
|
|
280
|
+
await uploadSingleFile(client, AwsS3, localDir + relFile, remoteDir + relFile, dryRun);
|
|
281
|
+
successCount++;
|
|
282
|
+
}
|
|
283
|
+
} else {
|
|
284
|
+
await uploadSingleFile(client, AwsS3, file.localPath, file.remotePath, dryRun);
|
|
285
|
+
successCount++;
|
|
286
|
+
}
|
|
269
287
|
} catch (error) {
|
|
270
288
|
console.error(` ❌ Failed to upload ${file.localPath}: ${error.message}`);
|
|
271
289
|
failCount++;
|
package/src/config.ts
CHANGED
package/src/fileSync.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { loadJwt } from "./login";
|
|
|
5
5
|
import { getConfig } from "./config";
|
|
6
6
|
import { services } from "./services";
|
|
7
7
|
import { S3Service } from "./services/S3";
|
|
8
|
+
import { getHashes, hasFileChangedSinceUpload, saveUploadHash, isLocalFileMatchingRemote } from "./hashes";
|
|
8
9
|
|
|
9
10
|
export interface FileSyncOptions {
|
|
10
11
|
upload?: boolean;
|
|
@@ -14,6 +15,33 @@ export interface FileSyncOptions {
|
|
|
14
15
|
dryRun?: boolean;
|
|
15
16
|
}
|
|
16
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Returns true if the path looks like a directory (ends with /)
|
|
20
|
+
*/
|
|
21
|
+
function isDirectoryPath(p: string): boolean {
|
|
22
|
+
return p.endsWith("/");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Recursively list all files in a local directory, returning relative paths
|
|
27
|
+
*/
|
|
28
|
+
function listFilesRecursively(dir: string): string[] {
|
|
29
|
+
const results: string[] = [];
|
|
30
|
+
if (!fs.existsSync(dir)) return results;
|
|
31
|
+
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
32
|
+
for (const entry of entries) {
|
|
33
|
+
if (entry.isDirectory()) {
|
|
34
|
+
const subFiles = listFilesRecursively(path.join(dir, entry.name));
|
|
35
|
+
for (const f of subFiles) {
|
|
36
|
+
results.push(entry.name + "/" + f);
|
|
37
|
+
}
|
|
38
|
+
} else {
|
|
39
|
+
results.push(entry.name);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return results;
|
|
43
|
+
}
|
|
44
|
+
|
|
17
45
|
/**
|
|
18
46
|
* Sync files between local filesystem and Knowhow FS
|
|
19
47
|
*/
|
|
@@ -67,11 +95,21 @@ export async function fileSync(options: FileSyncOptions = {}) {
|
|
|
67
95
|
|
|
68
96
|
try {
|
|
69
97
|
if (actualDirection === "download") {
|
|
70
|
-
|
|
71
|
-
|
|
98
|
+
if (isDirectoryPath(remotePath) || isDirectoryPath(localPath)) {
|
|
99
|
+
const count = await downloadDirectory(client, AwsS3, remotePath, localPath, dryRun);
|
|
100
|
+
successCount += count;
|
|
101
|
+
} else {
|
|
102
|
+
await downloadFile(client, AwsS3, remotePath, localPath, dryRun);
|
|
103
|
+
successCount++;
|
|
104
|
+
}
|
|
72
105
|
} else if (actualDirection === "upload") {
|
|
73
|
-
|
|
74
|
-
|
|
106
|
+
if (isDirectoryPath(remotePath) || isDirectoryPath(localPath)) {
|
|
107
|
+
const count = await uploadDirectory(client, AwsS3, remotePath, localPath, dryRun);
|
|
108
|
+
successCount += count;
|
|
109
|
+
} else {
|
|
110
|
+
await uploadFile(client, AwsS3, remotePath, localPath, dryRun);
|
|
111
|
+
successCount++;
|
|
112
|
+
}
|
|
75
113
|
}
|
|
76
114
|
} catch (error) {
|
|
77
115
|
console.error(`❌ Failed to sync ${remotePath}: ${error.message}`);
|
|
@@ -88,6 +126,7 @@ export async function fileSync(options: FileSyncOptions = {}) {
|
|
|
88
126
|
}
|
|
89
127
|
}
|
|
90
128
|
|
|
129
|
+
|
|
91
130
|
/**
|
|
92
131
|
* Download a file from Knowhow FS to local filesystem
|
|
93
132
|
*/
|
|
@@ -106,10 +145,14 @@ async function downloadFile(
|
|
|
106
145
|
}
|
|
107
146
|
|
|
108
147
|
try {
|
|
109
|
-
// Get presigned download URL
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
148
|
+
// Get presigned download URL + remote checksum
|
|
149
|
+
const { downloadUrl, checksumSHA256 } = await client.getOrgFilePresignedDownloadUrl(remotePath);
|
|
150
|
+
|
|
151
|
+
// Skip if local file matches remote checksum
|
|
152
|
+
if (isLocalFileMatchingRemote(localPath, checksumSHA256)) {
|
|
153
|
+
console.log(` ✓ Skipping ${localPath} (matches remote checksum)`);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
113
156
|
|
|
114
157
|
// Ensure parent directory exists
|
|
115
158
|
const dir = path.dirname(localPath);
|
|
@@ -118,7 +161,7 @@ async function downloadFile(
|
|
|
118
161
|
}
|
|
119
162
|
|
|
120
163
|
// Download file using presigned URL
|
|
121
|
-
await s3Service.downloadFromPresignedUrl(
|
|
164
|
+
await s3Service.downloadFromPresignedUrl(downloadUrl, localPath);
|
|
122
165
|
|
|
123
166
|
// Get file size for logging
|
|
124
167
|
const stats = fs.statSync(localPath);
|
|
@@ -151,6 +194,14 @@ async function uploadFile(
|
|
|
151
194
|
return;
|
|
152
195
|
}
|
|
153
196
|
|
|
197
|
+
// Skip upload if file hasn't changed since last upload
|
|
198
|
+
const hashes = await getHashes();
|
|
199
|
+
const changed = await hasFileChangedSinceUpload(localPath, hashes);
|
|
200
|
+
if (!changed) {
|
|
201
|
+
console.log(` ✓ Skipping ${localPath} (unchanged since last upload)`);
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
|
|
154
205
|
// Get presigned upload URL
|
|
155
206
|
const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
|
|
156
207
|
|
|
@@ -160,6 +211,99 @@ async function uploadFile(
|
|
|
160
211
|
// Notify backend that upload is complete to update the updatedAt timestamp
|
|
161
212
|
await client.markOrgFileUploadComplete(remotePath);
|
|
162
213
|
|
|
214
|
+
// Save upload hash so we can skip unchanged files next time
|
|
215
|
+
await saveUploadHash(localPath);
|
|
216
|
+
|
|
163
217
|
const stats = fs.statSync(localPath);
|
|
164
218
|
console.log(` ✓ Uploaded ${stats.size} bytes`);
|
|
165
219
|
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Upload all files from a local directory to a remote directory path
|
|
223
|
+
*/
|
|
224
|
+
export async function uploadDirectory(
|
|
225
|
+
client: KnowhowSimpleClient,
|
|
226
|
+
s3Service: S3Service,
|
|
227
|
+
remotePath: string,
|
|
228
|
+
localPath: string,
|
|
229
|
+
dryRun: boolean
|
|
230
|
+
): Promise<number> {
|
|
231
|
+
// Normalize paths to end with /
|
|
232
|
+
const remoteDir = remotePath.endsWith("/") ? remotePath : remotePath + "/";
|
|
233
|
+
const localDir = localPath.endsWith("/") ? localPath : localPath + "/";
|
|
234
|
+
|
|
235
|
+
console.log(`⬆️ Uploading directory ${localDir} → ${remoteDir}`);
|
|
236
|
+
|
|
237
|
+
if (!fs.existsSync(localDir)) {
|
|
238
|
+
console.warn(` ⚠️ Local directory not found: ${localDir}`);
|
|
239
|
+
return 0;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Find all files recursively in the local directory
|
|
243
|
+
const localFiles = listFilesRecursively(localDir);
|
|
244
|
+
|
|
245
|
+
if (localFiles.length === 0) {
|
|
246
|
+
console.log(` ⚠️ No local files found under ${localDir}`);
|
|
247
|
+
return 0;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
console.log(` Found ${localFiles.length} local file(s)`);
|
|
251
|
+
|
|
252
|
+
let count = 0;
|
|
253
|
+
for (const relFile of localFiles) {
|
|
254
|
+
const localFilePath = localDir + relFile;
|
|
255
|
+
const remoteFilePath = remoteDir + relFile;
|
|
256
|
+
await uploadFile(client, s3Service, remoteFilePath, localFilePath, dryRun);
|
|
257
|
+
count++;
|
|
258
|
+
}
|
|
259
|
+
return count;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Download all files from a remote directory path to a local directory
|
|
264
|
+
*/
|
|
265
|
+
async function downloadDirectory(
|
|
266
|
+
client: KnowhowSimpleClient,
|
|
267
|
+
s3Service: S3Service,
|
|
268
|
+
remotePath: string,
|
|
269
|
+
localPath: string,
|
|
270
|
+
dryRun: boolean
|
|
271
|
+
): Promise<number> {
|
|
272
|
+
// Normalize paths to end with /
|
|
273
|
+
const remoteDir = remotePath.endsWith("/") ? remotePath : remotePath + "/";
|
|
274
|
+
const localDir = localPath.endsWith("/") ? localPath : localPath + "/";
|
|
275
|
+
|
|
276
|
+
console.log(`⬇️ Downloading directory ${remoteDir} → ${localDir}`);
|
|
277
|
+
|
|
278
|
+
// List all org files and find those in the remote directory
|
|
279
|
+
const response = await client.listOrgFiles();
|
|
280
|
+
const allFiles = response.data;
|
|
281
|
+
|
|
282
|
+
// Find files where the full path starts with remoteDir
|
|
283
|
+
const matchingFiles = allFiles.filter((f) => {
|
|
284
|
+
const fullPath = f.folderPath.endsWith("/")
|
|
285
|
+
? f.folderPath + f.fileName
|
|
286
|
+
: f.folderPath + "/" + f.fileName;
|
|
287
|
+
return fullPath.startsWith(remoteDir);
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
if (matchingFiles.length === 0) {
|
|
291
|
+
console.log(` ⚠️ No remote files found under ${remoteDir}`);
|
|
292
|
+
return 0;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
console.log(` Found ${matchingFiles.length} remote file(s)`);
|
|
296
|
+
|
|
297
|
+
let count = 0;
|
|
298
|
+
for (const f of matchingFiles) {
|
|
299
|
+
const fullRemotePath = f.folderPath.endsWith("/")
|
|
300
|
+
? f.folderPath + f.fileName
|
|
301
|
+
: f.folderPath + "/" + f.fileName;
|
|
302
|
+
// Strip the base remote dir prefix to get relative path
|
|
303
|
+
const relativePath = fullRemotePath.slice(remoteDir.length);
|
|
304
|
+
const localFilePath = localDir + relativePath;
|
|
305
|
+
await downloadFile(client, s3Service, fullRemotePath, localFilePath, dryRun);
|
|
306
|
+
count++;
|
|
307
|
+
}
|
|
308
|
+
return count;
|
|
309
|
+
}
|
package/src/hashes.ts
CHANGED
|
@@ -70,3 +70,55 @@ export async function saveAllFileHashes(files: string[], promptHash: string) {
|
|
|
70
70
|
|
|
71
71
|
await saveHashes(hashes);
|
|
72
72
|
}
|
|
73
|
+
|
|
74
|
+
const UPLOAD_KEY = "upload";
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Returns true if the file has changed since the last successful upload
|
|
78
|
+
* (or if it has never been uploaded before)
|
|
79
|
+
*/
|
|
80
|
+
export async function hasFileChangedSinceUpload(
|
|
81
|
+
localPath: string,
|
|
82
|
+
hashes: any
|
|
83
|
+
): Promise<boolean> {
|
|
84
|
+
if (!fs.existsSync(localPath)) return true;
|
|
85
|
+
const content = fs.readFileSync(localPath);
|
|
86
|
+
const currentHash = crypto.createHash("md5").update(content).digest("hex");
|
|
87
|
+
return hashes[localPath]?.[UPLOAD_KEY] !== currentHash;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Saves the hash of the file at the time of a successful upload
|
|
92
|
+
*/
|
|
93
|
+
export async function saveUploadHash(localPath: string) {
|
|
94
|
+
const hashes = await getHashes();
|
|
95
|
+
const content = fs.readFileSync(localPath);
|
|
96
|
+
const currentHash = crypto.createHash("md5").update(content).digest("hex");
|
|
97
|
+
if (!hashes[localPath]) {
|
|
98
|
+
hashes[localPath] = { fileHash: currentHash, promptHash: "" };
|
|
99
|
+
}
|
|
100
|
+
hashes[localPath][UPLOAD_KEY] = currentHash;
|
|
101
|
+
await saveHashes(hashes);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Compute SHA-256 of a local file, returned as base64 (matches S3 encoding)
|
|
106
|
+
*/
|
|
107
|
+
export function computeSHA256Base64(filePath: string): string {
|
|
108
|
+
const content = fs.readFileSync(filePath);
|
|
109
|
+
return crypto.createHash("sha256").update(content).digest("base64");
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Returns true if the local file's SHA-256 matches the remote checksum,
|
|
114
|
+
* meaning the file is up-to-date and download can be skipped.
|
|
115
|
+
*/
|
|
116
|
+
export function isLocalFileMatchingRemote(
|
|
117
|
+
localPath: string,
|
|
118
|
+
remoteChecksumSHA256: string | null
|
|
119
|
+
): boolean {
|
|
120
|
+
if (!remoteChecksumSHA256) return false;
|
|
121
|
+
if (!fs.existsSync(localPath)) return false;
|
|
122
|
+
const localChecksum = computeSHA256Base64(localPath);
|
|
123
|
+
return localChecksum === remoteChecksumSHA256;
|
|
124
|
+
}
|
|
@@ -29,6 +29,12 @@ export class AgentSyncFs {
|
|
|
29
29
|
private agent: BaseAgent | undefined;
|
|
30
30
|
private threadUpdateHandler: ((...args: any[]) => void) | undefined;
|
|
31
31
|
private doneHandler: ((...args: any[]) => void) | undefined;
|
|
32
|
+
/**
|
|
33
|
+
* Tracks the most recent in-flight filesystem metadata update.
|
|
34
|
+
* The done handler awaits this before finalizing, preventing a race where
|
|
35
|
+
* the completion call writes before the last thread sync finishes.
|
|
36
|
+
*/
|
|
37
|
+
private pendingThreadUpdatePromise: Promise<void> | null = null;
|
|
32
38
|
|
|
33
39
|
constructor() {
|
|
34
40
|
// Start cleanup process when created
|
|
@@ -294,8 +300,12 @@ export class AgentSyncFs {
|
|
|
294
300
|
if (!this.taskId) return;
|
|
295
301
|
|
|
296
302
|
try {
|
|
297
|
-
await
|
|
298
|
-
|
|
303
|
+
// Track the pending update so the done handler can await it.
|
|
304
|
+
this.pendingThreadUpdatePromise = (async () => {
|
|
305
|
+
await this.updateMetadata(agent, true);
|
|
306
|
+
await this.checkForChanges(agent);
|
|
307
|
+
})();
|
|
308
|
+
await this.pendingThreadUpdatePromise;
|
|
299
309
|
} catch (error) {
|
|
300
310
|
console.error(`❌ Error during threadUpdate sync:`, error);
|
|
301
311
|
}
|
|
@@ -314,6 +324,17 @@ export class AgentSyncFs {
|
|
|
314
324
|
// Store finalization promise so callers can await it (same pattern as AgentSyncKnowhowWeb)
|
|
315
325
|
this.finalizationPromise = (async () => {
|
|
316
326
|
try {
|
|
327
|
+
// Flush any in-flight thread update before finalizing.
|
|
328
|
+
// This prevents the race where a pending "inProgress: true" metadata write
|
|
329
|
+
// overwrites the finalization write.
|
|
330
|
+
if (this.pendingThreadUpdatePromise) {
|
|
331
|
+
console.log(`⏳ [AgentSyncFs] Awaiting pending thread update before finalizing...`);
|
|
332
|
+
await this.pendingThreadUpdatePromise.catch(() => {
|
|
333
|
+
// Ignore errors in pending update — we still want to finalize
|
|
334
|
+
});
|
|
335
|
+
this.pendingThreadUpdatePromise = null;
|
|
336
|
+
}
|
|
337
|
+
|
|
317
338
|
await this.updateMetadata(agent, false, result);
|
|
318
339
|
console.log(`✅ Completed filesystem sync for task: ${this.taskId}`);
|
|
319
340
|
await this.cleanup();
|
|
@@ -435,5 +456,6 @@ export class AgentSyncFs {
|
|
|
435
456
|
this.eventHandlersSetup = false;
|
|
436
457
|
this.lastInputContent = "";
|
|
437
458
|
this.finalizationPromise = null;
|
|
459
|
+
this.pendingThreadUpdatePromise = null;
|
|
438
460
|
}
|
|
439
461
|
}
|
|
@@ -36,6 +36,12 @@ export class AgentSyncKnowhowWeb {
|
|
|
36
36
|
private agent: BaseAgent | undefined;
|
|
37
37
|
private threadUpdateHandler: ((...args: any[]) => void) | undefined;
|
|
38
38
|
private doneHandler: ((...args: any[]) => void) | undefined;
|
|
39
|
+
/**
|
|
40
|
+
* Tracks the most recent in-flight thread update API call.
|
|
41
|
+
* The done handler awaits this before sending the finalization call,
|
|
42
|
+
* preventing a race where the completion overwrites a later in-progress update.
|
|
43
|
+
*/
|
|
44
|
+
private pendingThreadUpdatePromise: Promise<void> | null = null;
|
|
39
45
|
|
|
40
46
|
constructor(baseUrl: string = KNOWHOW_API_URL) {
|
|
41
47
|
this.baseUrl = baseUrl;
|
|
@@ -240,11 +246,15 @@ export class AgentSyncKnowhowWeb {
|
|
|
240
246
|
}
|
|
241
247
|
|
|
242
248
|
try {
|
|
243
|
-
//
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
249
|
+
// Track the pending thread update so the done handler can await it.
|
|
250
|
+
this.pendingThreadUpdatePromise = (async () => {
|
|
251
|
+
// Update task with current state
|
|
252
|
+
await this.updateChatTask(this.knowhowTaskId!, agent, true);
|
|
253
|
+
|
|
254
|
+
// Check for pending messages, pause, or kill status
|
|
255
|
+
await this.checkAndProcessPendingMessages(agent, this.knowhowTaskId!);
|
|
256
|
+
})();
|
|
257
|
+
await this.pendingThreadUpdatePromise;
|
|
248
258
|
} catch (error) {
|
|
249
259
|
console.error(`❌ Error during threadUpdate sync:`, error);
|
|
250
260
|
// Continue execution even if synchronization fails
|
|
@@ -264,6 +274,17 @@ export class AgentSyncKnowhowWeb {
|
|
|
264
274
|
// Create a promise that tracks finalization
|
|
265
275
|
this.finalizationPromise = (async () => {
|
|
266
276
|
try {
|
|
277
|
+
// Flush any in-flight thread update before sending the completion call.
|
|
278
|
+
// This prevents the race where a "inProgress: true" update overtakes
|
|
279
|
+
// the "inProgress: false" finalization call.
|
|
280
|
+
if (this.pendingThreadUpdatePromise) {
|
|
281
|
+
console.log(`⏳ [AgentSync] Awaiting pending thread update before finalizing...`);
|
|
282
|
+
await this.pendingThreadUpdatePromise.catch(() => {
|
|
283
|
+
// Ignore errors in the pending update — we still want to finalize
|
|
284
|
+
});
|
|
285
|
+
this.pendingThreadUpdatePromise = null;
|
|
286
|
+
}
|
|
287
|
+
|
|
267
288
|
console.log(
|
|
268
289
|
`Updating Knowhow chat task on completion..., ${this.knowhowTaskId}`
|
|
269
290
|
);
|
|
@@ -306,6 +327,7 @@ export class AgentSyncKnowhowWeb {
|
|
|
306
327
|
this.knowhowTaskId = undefined;
|
|
307
328
|
this.eventHandlersSetup = false;
|
|
308
329
|
this.finalizationPromise = null;
|
|
330
|
+
this.pendingThreadUpdatePromise = null;
|
|
309
331
|
}
|
|
310
332
|
|
|
311
333
|
/**
|
|
@@ -24,6 +24,30 @@ import {
|
|
|
24
24
|
} from "../clients";
|
|
25
25
|
import { Config } from "../types";
|
|
26
26
|
|
|
27
|
+
// Remote sync placeholder interfaces
|
|
28
|
+
export interface CreateSessionPlaceholderRequest {
|
|
29
|
+
title?: string;
|
|
30
|
+
workerId?: string;
|
|
31
|
+
metadata?: Record<string, any>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface CreateSessionPlaceholderResponse {
|
|
35
|
+
sessionId: string;
|
|
36
|
+
orgId: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface CreateMessagePlaceholderRequest {
|
|
40
|
+
content: string;
|
|
41
|
+
agentName?: string;
|
|
42
|
+
modelName?: string;
|
|
43
|
+
metadata?: Record<string, any>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface CreateMessagePlaceholderResponse {
|
|
47
|
+
messageId: string;
|
|
48
|
+
taskId?: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
27
51
|
// Chat Task interfaces
|
|
28
52
|
export interface CreateMessageTaskRequest {
|
|
29
53
|
messageId: string;
|
|
@@ -575,7 +599,9 @@ export class KnowhowSimpleClient {
|
|
|
575
599
|
* Get presigned S3 URL for downloading a file from Knowhow FS.
|
|
576
600
|
* First finds or creates the file by path, then gets its download URL.
|
|
577
601
|
*/
|
|
578
|
-
async getOrgFilePresignedDownloadUrl(
|
|
602
|
+
async getOrgFilePresignedDownloadUrl(
|
|
603
|
+
filePath: string
|
|
604
|
+
): Promise<{ downloadUrl: string; checksumSHA256: string | null }> {
|
|
579
605
|
await this.checkJwt();
|
|
580
606
|
|
|
581
607
|
// Find the file by path
|
|
@@ -585,12 +611,15 @@ export class KnowhowSimpleClient {
|
|
|
585
611
|
}
|
|
586
612
|
|
|
587
613
|
// Get download URL using the file ID
|
|
588
|
-
const response = await http.post<{ downloadUrl: string }>(
|
|
614
|
+
const response = await http.post<{ downloadUrl: string; checksumSHA256: string | null }>(
|
|
589
615
|
`${this.baseUrl}/api/org-files/download/${file.id}`,
|
|
590
616
|
{},
|
|
591
617
|
{ headers: this.headers }
|
|
592
618
|
);
|
|
593
|
-
return
|
|
619
|
+
return {
|
|
620
|
+
downloadUrl: response.data.downloadUrl,
|
|
621
|
+
checksumSHA256: response.data.checksumSHA256 ?? null,
|
|
622
|
+
};
|
|
594
623
|
}
|
|
595
624
|
|
|
596
625
|
/**
|
|
@@ -692,4 +721,41 @@ export class KnowhowSimpleClient {
|
|
|
692
721
|
{ headers: this.headers }
|
|
693
722
|
);
|
|
694
723
|
}
|
|
724
|
+
|
|
725
|
+
// ============================================
|
|
726
|
+
// Remote Sync Placeholder Methods
|
|
727
|
+
// ============================================
|
|
728
|
+
|
|
729
|
+
/**
|
|
730
|
+
* Create a bare session stub without triggering AI inference.
|
|
731
|
+
* Used by the CLI remote sync feature to establish a remote session.
|
|
732
|
+
*/
|
|
733
|
+
async createSessionPlaceholder(
|
|
734
|
+
request: CreateSessionPlaceholderRequest = {}
|
|
735
|
+
): Promise<CreateSessionPlaceholderResponse> {
|
|
736
|
+
await this.checkJwt();
|
|
737
|
+
const response = await http.post<CreateSessionPlaceholderResponse>(
|
|
738
|
+
`${this.baseUrl}/api/chat/sessions/placeholder`,
|
|
739
|
+
request,
|
|
740
|
+
{ headers: this.headers }
|
|
741
|
+
);
|
|
742
|
+
return response.data;
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* Create a message placeholder in a session without triggering AI inference.
|
|
747
|
+
* Used by the CLI remote sync feature to register a message before syncing threads.
|
|
748
|
+
*/
|
|
749
|
+
async createMessagePlaceholder(
|
|
750
|
+
sessionId: string,
|
|
751
|
+
request: CreateMessagePlaceholderRequest
|
|
752
|
+
): Promise<CreateMessagePlaceholderResponse> {
|
|
753
|
+
await this.checkJwt();
|
|
754
|
+
const response = await http.post<CreateMessagePlaceholderResponse>(
|
|
755
|
+
`${this.baseUrl}/api/chat/sessions/${sessionId}/messages/placeholder`,
|
|
756
|
+
request,
|
|
757
|
+
{ headers: this.headers }
|
|
758
|
+
);
|
|
759
|
+
return response.data;
|
|
760
|
+
}
|
|
695
761
|
}
|
package/src/services/S3.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as fs from "fs";
|
|
2
|
+
import * as crypto from "crypto";
|
|
2
3
|
import { createWriteStream, createReadStream } from "fs";
|
|
3
4
|
import { pipeline, Readable } from "stream";
|
|
4
5
|
import * as util from "util";
|
|
@@ -11,19 +12,28 @@ export class S3Service {
|
|
|
11
12
|
filePath: string
|
|
12
13
|
): Promise<void> {
|
|
13
14
|
try {
|
|
14
|
-
const
|
|
15
|
+
const fileContent = fs.readFileSync(filePath);
|
|
15
16
|
const fileStats = await fs.promises.stat(filePath);
|
|
17
|
+
const sha256Base64 = crypto
|
|
18
|
+
.createHash("sha256")
|
|
19
|
+
.update(fileContent)
|
|
20
|
+
.digest("base64");
|
|
16
21
|
|
|
17
22
|
const response = await fetch(presignedUrl, {
|
|
18
23
|
method: "PUT",
|
|
19
|
-
headers: {
|
|
20
|
-
|
|
21
|
-
|
|
24
|
+
headers: {
|
|
25
|
+
"Content-Length": String(fileStats.size),
|
|
26
|
+
"x-amz-checksum-sha256": sha256Base64,
|
|
27
|
+
"x-amz-sdk-checksum-algorithm": "SHA256",
|
|
28
|
+
},
|
|
29
|
+
body: fileContent,
|
|
30
|
+
// @ts-ignore
|
|
22
31
|
duplex: "half",
|
|
23
32
|
});
|
|
24
33
|
|
|
25
34
|
if (!response.ok) {
|
|
26
|
-
|
|
35
|
+
const text = await response.text();
|
|
36
|
+
throw new Error(`Upload failed with status code: ${response.status} - ${text}`);
|
|
27
37
|
}
|
|
28
38
|
|
|
29
39
|
console.log("File uploaded successfully to pre-signed URL");
|
|
@@ -54,6 +54,7 @@ export class SessionManager {
|
|
|
54
54
|
sessionId: taskId,
|
|
55
55
|
knowhowMessageId: taskInfo.knowhowMessageId,
|
|
56
56
|
knowhowTaskId: taskInfo.knowhowTaskId,
|
|
57
|
+
chatSessionId: taskInfo.chatSessionId,
|
|
57
58
|
taskId,
|
|
58
59
|
agentName: taskInfo.agentName,
|
|
59
60
|
initialInput: taskInfo.initialInput,
|
|
@@ -99,6 +100,7 @@ export class SessionManager {
|
|
|
99
100
|
// Update Knowhow task fields if they exist in TaskInfo
|
|
100
101
|
session.knowhowMessageId = taskInfo.knowhowMessageId;
|
|
101
102
|
session.knowhowTaskId = taskInfo.knowhowTaskId;
|
|
103
|
+
session.chatSessionId = taskInfo.chatSessionId;
|
|
102
104
|
}
|
|
103
105
|
|
|
104
106
|
fs.writeFileSync(sessionPath, JSON.stringify(session, null, 2));
|