@juspay/neurolink 9.94.0 → 9.94.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/browser/neurolink.min.js +369 -371
- package/dist/cli/utils/audioPlayer.d.ts +44 -9
- package/dist/cli/utils/audioPlayer.js +162 -65
- package/dist/lib/rag/document/loaders.d.ts +12 -1
- package/dist/lib/rag/document/loaders.js +46 -32
- package/dist/lib/skills/skillMatcher.d.ts +7 -1
- package/dist/lib/skills/skillMatcher.js +23 -8
- package/dist/lib/types/cli.d.ts +9 -0
- package/dist/lib/utils/csvProcessor.d.ts +61 -3
- package/dist/lib/utils/csvProcessor.js +125 -40
- package/dist/lib/utils/csvUtils.d.ts +26 -0
- package/dist/lib/utils/csvUtils.js +85 -0
- package/dist/rag/document/loaders.d.ts +12 -1
- package/dist/rag/document/loaders.js +46 -32
- package/dist/skills/skillMatcher.d.ts +7 -1
- package/dist/skills/skillMatcher.js +23 -8
- package/dist/types/cli.d.ts +9 -0
- package/dist/utils/csvProcessor.d.ts +61 -3
- package/dist/utils/csvProcessor.js +125 -40
- package/dist/utils/csvUtils.d.ts +26 -0
- package/dist/utils/csvUtils.js +84 -0
- package/package.json +1 -1
|
@@ -6,32 +6,67 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module cli/utils/audioPlayer
|
|
8
8
|
*/
|
|
9
|
-
import type {
|
|
9
|
+
import type { TTSAudioFormat, CliAudioPlayerCommand } from "../../lib/types/index.js";
|
|
10
10
|
/**
|
|
11
11
|
* Get the file extension for an audio format
|
|
12
12
|
*
|
|
13
13
|
* @param format - Audio format
|
|
14
14
|
* @returns File extension string (e.g., "mp3", "wav")
|
|
15
15
|
*/
|
|
16
|
-
export declare function getAudioExtension(format:
|
|
16
|
+
export declare function getAudioExtension(format: TTSAudioFormat): string;
|
|
17
|
+
/**
|
|
18
|
+
* Escape a value for interpolation inside a single-quoted PowerShell string.
|
|
19
|
+
* PowerShell's single-quoted strings only treat `''` as a literal `'` — unlike
|
|
20
|
+
* shell/execFile argv, this string is parsed by powershell.exe itself, so an
|
|
21
|
+
* unescaped quote (e.g. a Windows username like `O'Brien` under `%TEMP%`)
|
|
22
|
+
* would break out of the string and let the remainder run as PS code.
|
|
23
|
+
*/
|
|
24
|
+
export declare function escapePowerShellSingleQuoted(value: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* Build the ordered list of player commands to try for a file.
|
|
27
|
+
*
|
|
28
|
+
* The list is tried in order until one succeeds; a missing binary (ENOENT) or a
|
|
29
|
+
* decode failure simply advances to the next candidate.
|
|
30
|
+
*
|
|
31
|
+
* The Linux ordering is the crux of the mp3-playback fix (#1138): `paplay`
|
|
32
|
+
* (PulseAudio) and `aplay` (ALSA) only decode libsndfile/PCM formats and
|
|
33
|
+
* **cannot** decode mp3/AAC. Since `--tts-format` defaults to mp3, compressed
|
|
34
|
+
* formats must lead with real decoders (ffplay/mpv/mpg123/cvlc); paplay/aplay
|
|
35
|
+
* remain only as last-resort fallbacks (and still work for wav/ogg).
|
|
36
|
+
*
|
|
37
|
+
* @param filePath - Path to the audio file
|
|
38
|
+
* @param format - Audio format
|
|
39
|
+
* @param platform - Platform id (defaults to `process.platform`; injectable for tests)
|
|
40
|
+
* @returns Ordered player commands; empty for unsupported platforms
|
|
41
|
+
*/
|
|
42
|
+
export declare function getPlayerCandidates(filePath: string, format: TTSAudioFormat, platform?: NodeJS.Platform): CliAudioPlayerCommand[];
|
|
43
|
+
/**
|
|
44
|
+
* Build a helpful, format-aware error message when no player could decode the file.
|
|
45
|
+
*
|
|
46
|
+
* Exported for testing: the message a user sees when Linux mp3 playback finds no
|
|
47
|
+
* decoder must name the real fix (ffmpeg/mpv/mpg123/VLC or `--tts-format wav`),
|
|
48
|
+
* not the misleading "install PulseAudio" that paplay-only routing produced (#1138).
|
|
49
|
+
*/
|
|
50
|
+
export declare function buildPlaybackErrorMessage(platform: NodeJS.Platform, format: TTSAudioFormat, attempts: string[]): string;
|
|
17
51
|
/**
|
|
18
52
|
* Play audio from a buffer using platform-specific CLI tools
|
|
19
53
|
*
|
|
20
|
-
* Writes the buffer to a temporary file, plays it using the
|
|
21
|
-
* system audio player, and cleans up the temp file
|
|
54
|
+
* Writes the buffer to a temporary file, plays it using the first available
|
|
55
|
+
* system audio player that can decode the format, and cleans up the temp file.
|
|
22
56
|
*
|
|
23
57
|
* Supported platforms:
|
|
24
|
-
* - macOS:
|
|
25
|
-
* - Linux:
|
|
26
|
-
*
|
|
58
|
+
* - macOS: `afplay` (built-in; decodes mp3/wav/aac/flac)
|
|
59
|
+
* - Linux: real decoders (ffplay/mpv/mpg123/cvlc) for compressed formats,
|
|
60
|
+
* aplay/paplay for wav — falling through the list until one works
|
|
61
|
+
* - Windows: PowerShell SoundPlayer (wav) or WMPlayer.OCX (compressed)
|
|
27
62
|
*
|
|
28
63
|
* @param buffer - Audio data buffer
|
|
29
64
|
* @param format - Audio format (mp3, wav, ogg, opus)
|
|
30
|
-
* @throws Error if
|
|
65
|
+
* @throws Error if no available player can play the audio, or platform is unsupported
|
|
31
66
|
*
|
|
32
67
|
* @example
|
|
33
68
|
* ```typescript
|
|
34
69
|
* await playAudio(audioBuffer, "mp3");
|
|
35
70
|
* ```
|
|
36
71
|
*/
|
|
37
|
-
export declare function playAudio(buffer: Buffer, format:
|
|
72
|
+
export declare function playAudio(buffer: Buffer, format: TTSAudioFormat): Promise<void>;
|
|
@@ -7,11 +7,25 @@
|
|
|
7
7
|
* @module cli/utils/audioPlayer
|
|
8
8
|
*/
|
|
9
9
|
import { execFile } from "node:child_process";
|
|
10
|
+
import { randomUUID } from "node:crypto";
|
|
10
11
|
import fs from "node:fs";
|
|
11
12
|
import os from "node:os";
|
|
12
13
|
import path from "node:path";
|
|
13
14
|
import { promisify } from "node:util";
|
|
14
15
|
const execFileAsync = promisify(execFile);
|
|
16
|
+
/** Wall-clock cap on a single player invocation; a hung decoder must not block the CLI forever. */
|
|
17
|
+
const PLAYER_TIMEOUT_MS = 30_000;
|
|
18
|
+
/**
|
|
19
|
+
* Single source of truth for format -> temp-file-extension. Formats outside this
|
|
20
|
+
* playback-supported set (m4a, flac, webm, mp4, mpeg, mpga, pcm16) fall back to
|
|
21
|
+
* "mp3" — `playAudio` only ever receives the four formats below in practice.
|
|
22
|
+
*/
|
|
23
|
+
const AUDIO_EXTENSIONS = {
|
|
24
|
+
mp3: "mp3",
|
|
25
|
+
wav: "wav",
|
|
26
|
+
ogg: "ogg",
|
|
27
|
+
opus: "opus",
|
|
28
|
+
};
|
|
15
29
|
/**
|
|
16
30
|
* Get the file extension for an audio format
|
|
17
31
|
*
|
|
@@ -19,73 +33,145 @@ const execFileAsync = promisify(execFile);
|
|
|
19
33
|
* @returns File extension string (e.g., "mp3", "wav")
|
|
20
34
|
*/
|
|
21
35
|
export function getAudioExtension(format) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return "opus";
|
|
31
|
-
default:
|
|
32
|
-
return "mp3";
|
|
33
|
-
}
|
|
36
|
+
return AUDIO_EXTENSIONS[format] ?? "mp3";
|
|
37
|
+
}
|
|
38
|
+
/** ffmpeg's ffplay decodes every format we emit; quiet + auto-exit for CLI use. */
|
|
39
|
+
function ffplayCommand(filePath) {
|
|
40
|
+
return {
|
|
41
|
+
command: "ffplay",
|
|
42
|
+
args: ["-nodisp", "-autoexit", "-loglevel", "quiet", filePath],
|
|
43
|
+
};
|
|
34
44
|
}
|
|
35
45
|
/**
|
|
36
|
-
*
|
|
46
|
+
* Escape a value for interpolation inside a single-quoted PowerShell string.
|
|
47
|
+
* PowerShell's single-quoted strings only treat `''` as a literal `'` — unlike
|
|
48
|
+
* shell/execFile argv, this string is parsed by powershell.exe itself, so an
|
|
49
|
+
* unescaped quote (e.g. a Windows username like `O'Brien` under `%TEMP%`)
|
|
50
|
+
* would break out of the string and let the remainder run as PS code.
|
|
51
|
+
*/
|
|
52
|
+
export function escapePowerShellSingleQuoted(value) {
|
|
53
|
+
return value.replace(/'/g, "''");
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Build the ordered list of player commands to try for a file.
|
|
57
|
+
*
|
|
58
|
+
* The list is tried in order until one succeeds; a missing binary (ENOENT) or a
|
|
59
|
+
* decode failure simply advances to the next candidate.
|
|
60
|
+
*
|
|
61
|
+
* The Linux ordering is the crux of the mp3-playback fix (#1138): `paplay`
|
|
62
|
+
* (PulseAudio) and `aplay` (ALSA) only decode libsndfile/PCM formats and
|
|
63
|
+
* **cannot** decode mp3/AAC. Since `--tts-format` defaults to mp3, compressed
|
|
64
|
+
* formats must lead with real decoders (ffplay/mpv/mpg123/cvlc); paplay/aplay
|
|
65
|
+
* remain only as last-resort fallbacks (and still work for wav/ogg).
|
|
37
66
|
*
|
|
38
67
|
* @param filePath - Path to the audio file
|
|
39
68
|
* @param format - Audio format
|
|
40
|
-
* @
|
|
69
|
+
* @param platform - Platform id (defaults to `process.platform`; injectable for tests)
|
|
70
|
+
* @returns Ordered player commands; empty for unsupported platforms
|
|
41
71
|
*/
|
|
42
|
-
function
|
|
43
|
-
const
|
|
72
|
+
export function getPlayerCandidates(filePath, format, platform = process.platform) {
|
|
73
|
+
const isWav = format === "wav";
|
|
44
74
|
switch (platform) {
|
|
45
75
|
case "darwin":
|
|
46
|
-
|
|
76
|
+
// afplay decodes mp3/wav/aac/flac natively.
|
|
77
|
+
return [{ command: "afplay", args: [filePath] }];
|
|
47
78
|
case "linux":
|
|
48
|
-
if (
|
|
49
|
-
|
|
79
|
+
if (isWav) {
|
|
80
|
+
// WAV/PCM: ALSA and PulseAudio both decode it; ffplay as a backstop.
|
|
81
|
+
return [
|
|
82
|
+
{ command: "aplay", args: [filePath] },
|
|
83
|
+
{ command: "paplay", args: [filePath] },
|
|
84
|
+
ffplayCommand(filePath),
|
|
85
|
+
];
|
|
86
|
+
}
|
|
87
|
+
// Compressed formats (mp3/ogg/opus): lead with decoders that actually
|
|
88
|
+
// handle them, then fall back to paplay/aplay (which work for some
|
|
89
|
+
// formats but never mp3).
|
|
90
|
+
return [
|
|
91
|
+
ffplayCommand(filePath),
|
|
92
|
+
{
|
|
93
|
+
command: "mpv",
|
|
94
|
+
args: ["--no-video", "--really-quiet", filePath],
|
|
95
|
+
},
|
|
96
|
+
...(format === "mp3"
|
|
97
|
+
? [{ command: "mpg123", args: ["-q", filePath] }]
|
|
98
|
+
: []),
|
|
99
|
+
{
|
|
100
|
+
command: "cvlc",
|
|
101
|
+
args: ["--play-and-exit", "--no-loop", "--intf", "dummy", filePath],
|
|
102
|
+
},
|
|
103
|
+
{ command: "paplay", args: [filePath] },
|
|
104
|
+
{ command: "aplay", args: [filePath] },
|
|
105
|
+
];
|
|
106
|
+
case "win32": {
|
|
107
|
+
// filePath is embedded inside a single-quoted PS string below, so it must
|
|
108
|
+
// be escaped — a Windows username with an apostrophe (e.g. `O'Brien`,
|
|
109
|
+
// present in `%TEMP%`) would otherwise break out of the string.
|
|
110
|
+
const psPath = escapePowerShellSingleQuoted(filePath);
|
|
111
|
+
if (isWav) {
|
|
112
|
+
return [
|
|
113
|
+
{
|
|
114
|
+
command: "powershell",
|
|
115
|
+
args: [
|
|
116
|
+
"-NoProfile",
|
|
117
|
+
"-Command",
|
|
118
|
+
`(New-Object System.Media.SoundPlayer '${psPath}').PlaySync()`,
|
|
119
|
+
],
|
|
120
|
+
},
|
|
121
|
+
];
|
|
50
122
|
}
|
|
51
|
-
return
|
|
52
|
-
|
|
53
|
-
if (format === "wav") {
|
|
54
|
-
return {
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
55
125
|
command: "powershell",
|
|
56
126
|
args: [
|
|
57
127
|
"-NoProfile",
|
|
58
128
|
"-Command",
|
|
59
|
-
|
|
129
|
+
`$player = New-Object -ComObject WMPlayer.OCX; $player.URL = '${psPath}'; $player.controls.play(); Start-Sleep -Seconds 1; while ($player.playState -eq 3) { Start-Sleep -Milliseconds 100 }; $player.close()`,
|
|
60
130
|
],
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
command: "powershell",
|
|
65
|
-
args: [
|
|
66
|
-
"-NoProfile",
|
|
67
|
-
"-Command",
|
|
68
|
-
`$player = New-Object -ComObject WMPlayer.OCX; $player.URL = '${filePath}'; $player.controls.play(); Start-Sleep -Seconds 1; while ($player.playState -eq 3) { Start-Sleep -Milliseconds 100 }; $player.close()`,
|
|
69
|
-
],
|
|
70
|
-
};
|
|
131
|
+
},
|
|
132
|
+
];
|
|
133
|
+
}
|
|
71
134
|
default:
|
|
72
|
-
|
|
135
|
+
return [];
|
|
73
136
|
}
|
|
74
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* Build a helpful, format-aware error message when no player could decode the file.
|
|
140
|
+
*
|
|
141
|
+
* Exported for testing: the message a user sees when Linux mp3 playback finds no
|
|
142
|
+
* decoder must name the real fix (ffmpeg/mpv/mpg123/VLC or `--tts-format wav`),
|
|
143
|
+
* not the misleading "install PulseAudio" that paplay-only routing produced (#1138).
|
|
144
|
+
*/
|
|
145
|
+
export function buildPlaybackErrorMessage(platform, format, attempts) {
|
|
146
|
+
const tried = attempts.length ? ` (tried: ${attempts.join("; ")})` : "";
|
|
147
|
+
if (platform === "linux") {
|
|
148
|
+
if (format === "wav") {
|
|
149
|
+
return `Could not play wav audio${tried}. Install ALSA (aplay) or PulseAudio (paplay).`;
|
|
150
|
+
}
|
|
151
|
+
// mpg123 only decodes mp3 (see getPlayerCandidates) — recommending it for
|
|
152
|
+
// ogg/opus failures would send users chasing a decoder that can't help.
|
|
153
|
+
if (format === "mp3") {
|
|
154
|
+
return `Could not play mp3 audio${tried}. paplay/aplay cannot decode mp3; install one of: ffmpeg (ffplay), mpv, mpg123, or VLC (cvlc) — or use --tts-format wav.`;
|
|
155
|
+
}
|
|
156
|
+
return `Could not play ${format} audio${tried}. Install ffmpeg (ffplay), mpv, or VLC (cvlc) — or use --tts-format wav.`;
|
|
157
|
+
}
|
|
158
|
+
return `Could not play ${format} audio${tried}. Ensure a system audio player is installed and available in PATH.`;
|
|
159
|
+
}
|
|
75
160
|
/**
|
|
76
161
|
* Play audio from a buffer using platform-specific CLI tools
|
|
77
162
|
*
|
|
78
|
-
* Writes the buffer to a temporary file, plays it using the
|
|
79
|
-
* system audio player, and cleans up the temp file
|
|
163
|
+
* Writes the buffer to a temporary file, plays it using the first available
|
|
164
|
+
* system audio player that can decode the format, and cleans up the temp file.
|
|
80
165
|
*
|
|
81
166
|
* Supported platforms:
|
|
82
|
-
* - macOS:
|
|
83
|
-
* - Linux:
|
|
84
|
-
*
|
|
167
|
+
* - macOS: `afplay` (built-in; decodes mp3/wav/aac/flac)
|
|
168
|
+
* - Linux: real decoders (ffplay/mpv/mpg123/cvlc) for compressed formats,
|
|
169
|
+
* aplay/paplay for wav — falling through the list until one works
|
|
170
|
+
* - Windows: PowerShell SoundPlayer (wav) or WMPlayer.OCX (compressed)
|
|
85
171
|
*
|
|
86
172
|
* @param buffer - Audio data buffer
|
|
87
173
|
* @param format - Audio format (mp3, wav, ogg, opus)
|
|
88
|
-
* @throws Error if
|
|
174
|
+
* @throws Error if no available player can play the audio, or platform is unsupported
|
|
89
175
|
*
|
|
90
176
|
* @example
|
|
91
177
|
* ```typescript
|
|
@@ -94,36 +180,47 @@ function getPlayerCommand(filePath, format) {
|
|
|
94
180
|
*/
|
|
95
181
|
export async function playAudio(buffer, format) {
|
|
96
182
|
const ext = getAudioExtension(format);
|
|
97
|
-
|
|
183
|
+
// randomUUID (not just Date.now()) avoids two concurrent calls in the same
|
|
184
|
+
// millisecond racing on the same temp path.
|
|
185
|
+
const tempFile = path.join(os.tmpdir(), `nl-tts-${randomUUID()}.${ext}`);
|
|
98
186
|
try {
|
|
99
187
|
// Write audio buffer to temp file
|
|
100
188
|
await fs.promises.writeFile(tempFile, buffer);
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
189
|
+
const candidates = getPlayerCandidates(tempFile, format);
|
|
190
|
+
if (candidates.length === 0) {
|
|
191
|
+
throw new Error(`Unsupported platform: ${process.platform}. Audio playback is supported on macOS, Linux, and Windows.`);
|
|
104
192
|
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
193
|
+
const attempts = [];
|
|
194
|
+
for (const { command, args } of candidates) {
|
|
195
|
+
try {
|
|
196
|
+
// `timeout` guards against a hung decoder (e.g. blocked on the audio
|
|
197
|
+
// device) so playback can't stall the CLI forever; SIGTERM lets the
|
|
198
|
+
// player shut down cleanly before Node force-kills it.
|
|
199
|
+
await execFileAsync(command, args, {
|
|
200
|
+
timeout: PLAYER_TIMEOUT_MS,
|
|
201
|
+
killSignal: "SIGTERM",
|
|
202
|
+
});
|
|
203
|
+
return; // Played successfully.
|
|
204
|
+
}
|
|
205
|
+
catch (execError) {
|
|
206
|
+
const err = execError;
|
|
207
|
+
// Every failure (missing binary, decode error, or timeout) is
|
|
208
|
+
// recorded and surfaced in the final error below — none are lost.
|
|
209
|
+
// We still advance to the next candidate rather than aborting on the
|
|
210
|
+
// first non-ENOENT failure: `command` existing but failing on this
|
|
211
|
+
// particular file (e.g. a corrupt stream, or one decoder lacking
|
|
212
|
+
// codec support) doesn't mean the *next* candidate will fail too,
|
|
213
|
+
// and failing fast here would defeat the multi-decoder fallback
|
|
214
|
+
// that #1138 exists to provide.
|
|
215
|
+
const detail = err.killed
|
|
216
|
+
? `timed out after ${PLAYER_TIMEOUT_MS}ms`
|
|
217
|
+
: err.code === "ENOENT"
|
|
218
|
+
? "not installed"
|
|
219
|
+
: (err.message ?? "failed");
|
|
220
|
+
attempts.push(`${command}: ${detail}`);
|
|
124
221
|
}
|
|
125
|
-
throw execError;
|
|
126
222
|
}
|
|
223
|
+
throw new Error(buildPlaybackErrorMessage(process.platform, format, attempts));
|
|
127
224
|
}
|
|
128
225
|
finally {
|
|
129
226
|
// Always clean up temp file
|
|
@@ -61,9 +61,20 @@ export declare class JSONLoader extends TextLoader {
|
|
|
61
61
|
export declare class CSVLoader extends TextLoader {
|
|
62
62
|
load(source: string, options?: CSVLoaderOptions): Promise<MDocument>;
|
|
63
63
|
canHandle(source: string): boolean;
|
|
64
|
-
private parseCSVLine;
|
|
65
64
|
private toMarkdownTable;
|
|
66
65
|
private toTextTable;
|
|
66
|
+
/**
|
|
67
|
+
* Collapse embedded newlines to a single space so one logical CSV row
|
|
68
|
+
* renders as exactly one line in the text/markdown table output.
|
|
69
|
+
*
|
|
70
|
+
* The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
|
|
71
|
+
* preserves newlines embedded inside a quoted field verbatim — correct
|
|
72
|
+
* for the `json` output format (where `JSON.stringify` escapes them), but
|
|
73
|
+
* left as-is here it would split a single record across multiple
|
|
74
|
+
* physical table rows. Mirrors the same normalization `CSVProcessor`
|
|
75
|
+
* already applies in its own markdown formatter.
|
|
76
|
+
*/
|
|
77
|
+
private collapseEmbeddedNewlines;
|
|
67
78
|
}
|
|
68
79
|
/**
|
|
69
80
|
* PDF file loader
|
|
@@ -27,6 +27,8 @@ import { existsSync } from "fs";
|
|
|
27
27
|
import { readFile } from "fs/promises";
|
|
28
28
|
import { basename, extname } from "path";
|
|
29
29
|
import { logger } from "../../utils/logger.js";
|
|
30
|
+
import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, } from "../../utils/csvProcessor.js";
|
|
31
|
+
import { splitCSVFields } from "../../utils/csvUtils.js";
|
|
30
32
|
import { MDocument } from "./MDocument.js";
|
|
31
33
|
/**
|
|
32
34
|
* Text file loader
|
|
@@ -115,13 +117,28 @@ export class JSONLoader extends TextLoader {
|
|
|
115
117
|
export class CSVLoader extends TextLoader {
|
|
116
118
|
async load(source, options) {
|
|
117
119
|
const content = await this.loadContent(source, options?.encoding);
|
|
118
|
-
const { delimiter
|
|
119
|
-
|
|
120
|
+
const { delimiter, hasHeader = true, columns, outputFormat = "text", } = options || {};
|
|
121
|
+
// #361: an explicit delimiter always wins; otherwise detect it from the
|
|
122
|
+
// content + extension so a .tsv fed through RAG isn't collapsed into one
|
|
123
|
+
// comma-column (CSVLoader.canHandle already accepts .tsv).
|
|
124
|
+
const effectiveDelimiter = delimiter ?? CSVProcessor.detectDelimiter(content, extname(source));
|
|
125
|
+
// Route header/row parsing through the same shared, quote-aware path
|
|
126
|
+
// detectDelimiter() above already uses internally: a quote-aware logical
|
|
127
|
+
// row split (splitCsvLines — a newline inside a quoted field stays part
|
|
128
|
+
// of that field instead of breaking the row) with a leading Excel `sep=`
|
|
129
|
+
// preamble line stripped (stripMetadataLine), so it can't be parsed as
|
|
130
|
+
// the header. Field-level splitting also goes through the shared
|
|
131
|
+
// RFC-4180-aware splitCSVFields() instead of the loader's own naive
|
|
132
|
+
// backslash-escape parser, which didn't handle doubled-quote (`""`)
|
|
133
|
+
// escaping or quoted delimiters in the headerless-columns branch.
|
|
134
|
+
const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
|
|
135
|
+
const lines = strippedLines.filter((line) => line.trim());
|
|
120
136
|
const headers = hasHeader
|
|
121
|
-
?
|
|
122
|
-
: columns ||
|
|
137
|
+
? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
|
|
138
|
+
: columns ||
|
|
139
|
+
splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
|
|
123
140
|
const dataLines = hasHeader ? lines.slice(1) : lines;
|
|
124
|
-
const rows = dataLines.map((line) =>
|
|
141
|
+
const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
|
|
125
142
|
let formattedContent;
|
|
126
143
|
switch (outputFormat) {
|
|
127
144
|
case "json":
|
|
@@ -145,42 +162,39 @@ export class CSVLoader extends TextLoader {
|
|
|
145
162
|
const ext = extname(source).toLowerCase();
|
|
146
163
|
return ext === ".csv" || ext === ".tsv";
|
|
147
164
|
}
|
|
148
|
-
parseCSVLine(line, delimiter) {
|
|
149
|
-
const result = [];
|
|
150
|
-
let current = "";
|
|
151
|
-
let inQuotes = false;
|
|
152
|
-
for (let i = 0; i < line.length; i++) {
|
|
153
|
-
const char = line[i];
|
|
154
|
-
if (char === '"' && (i === 0 || line[i - 1] !== "\\")) {
|
|
155
|
-
inQuotes = !inQuotes;
|
|
156
|
-
}
|
|
157
|
-
else if (char === delimiter && !inQuotes) {
|
|
158
|
-
result.push(current.trim());
|
|
159
|
-
current = "";
|
|
160
|
-
}
|
|
161
|
-
else {
|
|
162
|
-
current += char;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
result.push(current.trim());
|
|
166
|
-
return result;
|
|
167
|
-
}
|
|
168
165
|
toMarkdownTable(headers, rows) {
|
|
169
|
-
const
|
|
170
|
-
const
|
|
171
|
-
const
|
|
166
|
+
const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
|
|
167
|
+
const headerRow = `| ${safeHeaders.join(" | ")} |`;
|
|
168
|
+
const separator = `| ${safeHeaders.map(() => "---").join(" | ")} |`;
|
|
169
|
+
const dataRows = rows.map((row) => `| ${row.map((cell) => this.collapseEmbeddedNewlines(cell)).join(" | ")} |`);
|
|
172
170
|
return [headerRow, separator, ...dataRows].join("\n");
|
|
173
171
|
}
|
|
174
172
|
toTextTable(headers, rows) {
|
|
175
|
-
const
|
|
176
|
-
const
|
|
173
|
+
const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
|
|
174
|
+
const safeRows = rows.map((row) => row.map((cell) => this.collapseEmbeddedNewlines(cell)));
|
|
175
|
+
const allRows = [safeHeaders, ...safeRows];
|
|
176
|
+
const colWidths = safeHeaders.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
|
|
177
177
|
const formatRow = (row) => row.map((cell, i) => (cell || "").padEnd(colWidths[i])).join(" | ");
|
|
178
178
|
return [
|
|
179
|
-
formatRow(
|
|
179
|
+
formatRow(safeHeaders),
|
|
180
180
|
colWidths.map((w) => "-".repeat(w)).join("-+-"),
|
|
181
|
-
...
|
|
181
|
+
...safeRows.map(formatRow),
|
|
182
182
|
].join("\n");
|
|
183
183
|
}
|
|
184
|
+
/**
|
|
185
|
+
* Collapse embedded newlines to a single space so one logical CSV row
|
|
186
|
+
* renders as exactly one line in the text/markdown table output.
|
|
187
|
+
*
|
|
188
|
+
* The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
|
|
189
|
+
* preserves newlines embedded inside a quoted field verbatim — correct
|
|
190
|
+
* for the `json` output format (where `JSON.stringify` escapes them), but
|
|
191
|
+
* left as-is here it would split a single record across multiple
|
|
192
|
+
* physical table rows. Mirrors the same normalization `CSVProcessor`
|
|
193
|
+
* already applies in its own markdown formatter.
|
|
194
|
+
*/
|
|
195
|
+
collapseEmbeddedNewlines(cell) {
|
|
196
|
+
return cell.replace(/\r\n|\n|\r/g, " ");
|
|
197
|
+
}
|
|
184
198
|
}
|
|
185
199
|
/**
|
|
186
200
|
* PDF file loader
|
|
@@ -26,7 +26,13 @@ export declare function filterSkillIndex(items: SkillIndexItem[], query: SkillSe
|
|
|
26
26
|
* keys are built by concatenation (S3, Redis) or path joining.
|
|
27
27
|
*/
|
|
28
28
|
export declare function isSafeSkillResourcePath(resourcePath: string): boolean;
|
|
29
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Whether a skill is visible from the calling scope. Fails closed, matching
|
|
31
|
+
* `filterSkillIndex`: a scoped skill is hidden unless the caller supplies a
|
|
32
|
+
* `scopeId` that the skill explicitly lists. Omitting `scopeId` therefore
|
|
33
|
+
* yields global skills only — it must never expose a scoped skill by name
|
|
34
|
+
* through `use_skill` / `read_skill_resource` (#1139).
|
|
35
|
+
*/
|
|
30
36
|
export declare function isSkillVisibleInScope(skill: Pick<SkillDefinition, "scope" | "scopeIds">, scopeId?: string): boolean;
|
|
31
37
|
/**
|
|
32
38
|
* Render the `<available_skills>` block embedded in the use_skill tool
|
|
@@ -29,11 +29,14 @@ export function filterSkillIndex(items, query) {
|
|
|
29
29
|
if ((item.status ?? "active") !== "active") {
|
|
30
30
|
return false;
|
|
31
31
|
}
|
|
32
|
-
// Scope filter: global skills always pass
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
|
|
36
|
-
|
|
32
|
+
// Scope filter: global skills always pass. Scoped skills require a
|
|
33
|
+
// scopeId that matches; when the caller supplies no scopeId we fail
|
|
34
|
+
// closed and exclude them, so a shared multi-tenant instance that forgets
|
|
35
|
+
// to pass a per-call scopeId never leaks one tenant's scoped skills to
|
|
36
|
+
// everyone (#1139). Callers that legitimately want a tenant's scoped
|
|
37
|
+
// skills must pass that tenant's scopeId.
|
|
38
|
+
if (item.scope === "scoped") {
|
|
39
|
+
if (!query.scopeId || !(item.scopeIds ?? []).includes(query.scopeId)) {
|
|
37
40
|
return false;
|
|
38
41
|
}
|
|
39
42
|
}
|
|
@@ -67,11 +70,20 @@ export function isSafeSkillResourcePath(resourcePath) {
|
|
|
67
70
|
!normalized.startsWith("/") &&
|
|
68
71
|
!normalized.split("/").some((segment) => segment === ".." || segment === ""));
|
|
69
72
|
}
|
|
70
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Whether a skill is visible from the calling scope. Fails closed, matching
|
|
75
|
+
* `filterSkillIndex`: a scoped skill is hidden unless the caller supplies a
|
|
76
|
+
* `scopeId` that the skill explicitly lists. Omitting `scopeId` therefore
|
|
77
|
+
* yields global skills only — it must never expose a scoped skill by name
|
|
78
|
+
* through `use_skill` / `read_skill_resource` (#1139).
|
|
79
|
+
*/
|
|
71
80
|
export function isSkillVisibleInScope(skill, scopeId) {
|
|
72
|
-
if (
|
|
81
|
+
if (skill.scope !== "scoped") {
|
|
73
82
|
return true;
|
|
74
83
|
}
|
|
84
|
+
if (!scopeId) {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
75
87
|
return (skill.scopeIds ?? []).includes(scopeId);
|
|
76
88
|
}
|
|
77
89
|
/** Trim a description to its first sentence (or the whole text when unsplittable). */
|
|
@@ -127,7 +139,10 @@ export function renderSkillListing(items, budgetChars) {
|
|
|
127
139
|
* Returns null when nothing is visible so callers can skip injection.
|
|
128
140
|
*/
|
|
129
141
|
export function formatSkillsPromptIndex(items, maxItems) {
|
|
130
|
-
|
|
142
|
+
// Nothing visible, or the index is explicitly disabled (maxItems <= 0):
|
|
143
|
+
// skip injection entirely rather than emit a header + "N more skills exist"
|
|
144
|
+
// note with zero skill lines (#1139).
|
|
145
|
+
if (items.length === 0 || maxItems <= 0) {
|
|
131
146
|
return null;
|
|
132
147
|
}
|
|
133
148
|
const visible = items.slice(0, maxItems);
|
package/dist/lib/types/cli.d.ts
CHANGED
|
@@ -1675,3 +1675,12 @@ export type CliNetworkCommandArgs = BaseCommandArgs & {
|
|
|
1675
1675
|
/** Show detailed information */
|
|
1676
1676
|
detailed?: boolean;
|
|
1677
1677
|
};
|
|
1678
|
+
/**
|
|
1679
|
+
* A single audio-player invocation for CLI TTS playback: a binary plus its
|
|
1680
|
+
* arguments for `execFile`. The player list is tried in order until one
|
|
1681
|
+
* succeeds (see `src/cli/utils/audioPlayer.ts`).
|
|
1682
|
+
*/
|
|
1683
|
+
export type CliAudioPlayerCommand = {
|
|
1684
|
+
command: string;
|
|
1685
|
+
args: string[];
|
|
1686
|
+
};
|