@pmndre/pine 0.0.0-stage → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +226 -2
- package/bin/pine.js +27 -0
- package/docs/SKILL.md +234 -0
- package/docs/references/agents.md +454 -0
- package/docs/references/coding.md +311 -0
- package/docs/references/computer.md +212 -0
- package/docs/references/configuration.md +310 -0
- package/docs/references/events.md +267 -0
- package/docs/references/mcp.md +420 -0
- package/docs/references/operations.md +394 -0
- package/docs/references/overview.md +195 -0
- package/docs/references/security.md +896 -0
- package/docs/references/setup.md +385 -0
- package/docs/references/sources.md +772 -0
- package/docs/references/stack.md +121 -0
- package/docs/references/tui.md +575 -0
- package/examples/home/.env.example +7 -0
- package/examples/home/USER.md +4 -0
- package/examples/home/agents/main/agent.json +4 -0
- package/examples/home/agents/main/workspace/AGENTS.md +14 -0
- package/examples/home/agents/main/workspace/MEMORY.md +4 -0
- package/examples/home/agents/main/workspace/SOUL.md +4 -0
- package/examples/home/models.json +62 -0
- package/examples/home/pine.json +34 -0
- package/examples/home/skills/README.txt +7 -0
- package/examples/launchd/com.pine.daemon.plist +32 -0
- package/examples/systemd/pine.service +22 -0
- package/mcp/browser.d.ts +99 -0
- package/mcp/browser.js +496 -0
- package/mcp/commands.d.ts +20 -0
- package/mcp/commands.js +88 -0
- package/mcp/computer.d.ts +139 -0
- package/mcp/computer.js +698 -0
- package/mcp/computer.swift +989 -0
- package/mcp/lib.d.ts +23 -0
- package/mcp/lib.js +52 -0
- package/mcp/web.d.ts +6 -0
- package/mcp/web.js +140 -0
- package/package.json +32 -4
- package/src/access.d.ts +112 -0
- package/src/access.js +438 -0
- package/src/actions.d.ts +75 -0
- package/src/actions.js +167 -0
- package/src/admin.d.ts +167 -0
- package/src/admin.js +625 -0
- package/src/agent-admin.d.ts +24 -0
- package/src/agent-admin.js +55 -0
- package/src/agents.d.ts +540 -0
- package/src/agents.js +2776 -0
- package/src/background.d.ts +29 -0
- package/src/background.js +100 -0
- package/src/cli.d.ts +2 -0
- package/src/cli.js +777 -0
- package/src/computer.d.ts +30 -0
- package/src/computer.js +32 -0
- package/src/config.d.ts +396 -0
- package/src/config.js +549 -0
- package/src/dev/claude-model.d.ts +37 -0
- package/src/dev/claude-model.js +204 -0
- package/src/env-file.d.ts +4 -0
- package/src/env-file.js +40 -0
- package/src/goals.d.ts +133 -0
- package/src/goals.js +316 -0
- package/src/http.d.ts +46 -0
- package/src/http.js +509 -0
- package/src/limiter.d.ts +32 -0
- package/src/limiter.js +84 -0
- package/src/login.d.ts +23 -0
- package/src/login.js +130 -0
- package/src/manual.d.ts +61 -0
- package/src/manual.js +443 -0
- package/src/mcp-config.d.ts +49 -0
- package/src/mcp-config.js +182 -0
- package/src/models.d.ts +90 -0
- package/src/models.js +274 -0
- package/src/net.d.ts +50 -0
- package/src/net.js +150 -0
- package/src/pine.d.ts +32 -0
- package/src/pine.js +134 -0
- package/src/programs.d.ts +123 -0
- package/src/programs.js +412 -0
- package/src/router.d.ts +56 -0
- package/src/router.js +164 -0
- package/src/sandbox-fs.d.ts +34 -0
- package/src/sandbox-fs.js +186 -0
- package/src/sandbox-linux.d.ts +96 -0
- package/src/sandbox-linux.js +297 -0
- package/src/sandbox-setup.d.ts +70 -0
- package/src/sandbox-setup.js +216 -0
- package/src/security.d.ts +238 -0
- package/src/security.js +660 -0
- package/src/skills.d.ts +67 -0
- package/src/skills.js +163 -0
- package/src/snapshots.d.ts +68 -0
- package/src/snapshots.js +306 -0
- package/src/source-types.d.ts +61 -0
- package/src/source-types.js +1 -0
- package/src/sources/command.d.ts +4 -0
- package/src/sources/command.js +77 -0
- package/src/sources/folder.d.ts +6 -0
- package/src/sources/folder.js +116 -0
- package/src/sources/ftp.d.ts +4 -0
- package/src/sources/ftp.js +307 -0
- package/src/sources/http.d.ts +4 -0
- package/src/sources/http.js +201 -0
- package/src/sources/mqtt.d.ts +18 -0
- package/src/sources/mqtt.js +97 -0
- package/src/sources/telegram.d.ts +22 -0
- package/src/sources/telegram.js +283 -0
- package/src/sources/webhook.d.ts +4 -0
- package/src/sources/webhook.js +52 -0
- package/src/sources.d.ts +344 -0
- package/src/sources.js +966 -0
- package/src/store.d.ts +99 -0
- package/src/store.js +277 -0
- package/src/timers.d.ts +83 -0
- package/src/timers.js +318 -0
- package/src/tools.d.ts +11 -0
- package/src/tools.js +1297 -0
- package/src/tui/app.d.ts +305 -0
- package/src/tui/app.js +3195 -0
- package/src/tui/client.d.ts +236 -0
- package/src/tui/client.js +131 -0
- package/src/tui/code-pane.d.ts +68 -0
- package/src/tui/code-pane.js +535 -0
- package/src/tui/highlight.d.ts +8 -0
- package/src/tui/highlight.js +86 -0
- package/src/tui/picture.d.ts +206 -0
- package/src/tui/picture.js +801 -0
- package/src/tui/run.d.ts +8 -0
- package/src/tui/run.js +97 -0
- package/src/tui/theme.d.ts +32 -0
- package/src/tui/theme.js +128 -0
- package/src/tui/undo.d.ts +5 -0
- package/src/tui/undo.js +50 -0
- package/src/tui/voice.d.ts +49 -0
- package/src/tui/voice.js +155 -0
- package/src/tui/voice.swift +223 -0
- package/src/types.d.ts +139 -0
- package/src/types.js +1 -0
- package/src/undo.d.ts +224 -0
- package/src/undo.js +465 -0
- package/src/update.d.ts +35 -0
- package/src/update.js +130 -0
- package/src/util.d.ts +65 -0
- package/src/util.js +171 -0
- package/src/watching.d.ts +20 -0
- package/src/watching.js +68 -0
- package/src/workspace-view.d.ts +72 -0
- package/src/workspace-view.js +221 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { TSchema } from "typebox";
|
|
2
|
+
/** What setting a source up would mean, worked out before anything is written. */
|
|
3
|
+
export interface SourcePlan {
|
|
4
|
+
/** The options as they go into the configuration: paths resolved, defaults that matter filled in, secrets made. */
|
|
5
|
+
options: Record<string, unknown>;
|
|
6
|
+
/** What it will do, in plain words, for the owner's dialog. */
|
|
7
|
+
does: string[];
|
|
8
|
+
/** It reaches something new (a folder elsewhere, a network connection, a port), so the owner is asked first. */
|
|
9
|
+
opens: boolean;
|
|
10
|
+
/** A folder the agent gets read access to along with the source. */
|
|
11
|
+
read?: string;
|
|
12
|
+
/** What the agent should pass on once it is set up (where a device finds its password, for example). */
|
|
13
|
+
tell?: string;
|
|
14
|
+
/**
|
|
15
|
+
* What it reports is the owner speaking, not outside content: the source itself makes sure of
|
|
16
|
+
* who is on the other end (a Telegram account the owner paired, for example). The owner is told
|
|
17
|
+
* so before they confirm.
|
|
18
|
+
*/
|
|
19
|
+
owner?: boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface SourceSetup {
|
|
22
|
+
/** The name the source will have. */
|
|
23
|
+
name: string;
|
|
24
|
+
/** What it is called in events and addresses: its name, or name@agent for an agent's own source. */
|
|
25
|
+
key: string;
|
|
26
|
+
/** The agent it is set up by, and that agent's workspace. */
|
|
27
|
+
agent: string;
|
|
28
|
+
workspace: string;
|
|
29
|
+
/** pine's home folder, which no source may point into. */
|
|
30
|
+
home: string;
|
|
31
|
+
/** An absolute path for one an agent wrote: ~ is the user's home, anything relative is meant from the workspace. */
|
|
32
|
+
resolve(p: string): string;
|
|
33
|
+
/** A fresh secret of `bytes` random bytes, as letters and digits. */
|
|
34
|
+
random(bytes: number): string;
|
|
35
|
+
/** The name a secret option gets in pine's .env file, to tell the owner where to find it. */
|
|
36
|
+
envName(option: string): string;
|
|
37
|
+
}
|
|
38
|
+
export interface SourceType {
|
|
39
|
+
/** One line: what a source of this type reports and what it is for. */
|
|
40
|
+
summary: string;
|
|
41
|
+
/**
|
|
42
|
+
* Its options as a JSON schema of an object (typebox's Type.Object). Give each property a
|
|
43
|
+
* description; mark one that is a password or token with `secret: true`, and pine keeps its value
|
|
44
|
+
* in .env and out of the configuration, the event log and what models see.
|
|
45
|
+
*/
|
|
46
|
+
options: TSchema;
|
|
47
|
+
/** Check the options and say what the source would do. Throw for what cannot work. Without it, a set-up is always put to the owner. */
|
|
48
|
+
plan?(options: Record<string, any>, setup: SourceSetup): SourcePlan;
|
|
49
|
+
/**
|
|
50
|
+
* Its "message" events are somebody writing to the agent: the event's `text` is what they
|
|
51
|
+
* wrote, it reaches the agent as a message rather than as a report, and the agent's answer goes
|
|
52
|
+
* back through the source.
|
|
53
|
+
*/
|
|
54
|
+
talks?: boolean;
|
|
55
|
+
/**
|
|
56
|
+
* It can also send. One line for the agent on what sending through it does and what to pass,
|
|
57
|
+
* for example 'a Telegram message to your owner: {"text": "..."}'. The agent's `send` tool
|
|
58
|
+
* offers every source that says this.
|
|
59
|
+
*/
|
|
60
|
+
sends?: string;
|
|
61
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { parseDuration } from "../util.js";
|
|
3
|
+
import { clip, found, harmless, lineDiff, pattern, shown } from "../watching.js";
|
|
4
|
+
/**
|
|
5
|
+
* A command, run again and again: reports when what it prints or how it ends changes. For what
|
|
6
|
+
* only a command can tell: free disk space, a build, a branch that moved, a log, whether a
|
|
7
|
+
* machine answers.
|
|
8
|
+
*
|
|
9
|
+
* options: { run, every?, match?, timeout? }
|
|
10
|
+
* run the command line, run by bash from the workspace of the agent the source belongs to
|
|
11
|
+
* every how often ("1m" by default)
|
|
12
|
+
* match watch only what this regular expression finds in what it prints
|
|
13
|
+
* timeout how long one run may take ("30s" by default)
|
|
14
|
+
*
|
|
15
|
+
* event "changed": { command, code, previousCode?, value, previous } with match
|
|
16
|
+
* { command, code, previousCode?, added, removed, more? } without
|
|
17
|
+
*
|
|
18
|
+
* The command runs in the sandbox of that agent's shell: it reaches what the agent reaches and
|
|
19
|
+
* nothing else. The first run is the starting point and reports nothing.
|
|
20
|
+
*/
|
|
21
|
+
const MIN = 5000;
|
|
22
|
+
export const type = {
|
|
23
|
+
summary: "a command run again and again: reports when what it prints, or how it ends (its exit code), changes. For what only a command can tell: free disk space, a build, a branch that moved, a log, whether a machine answers. It runs like a command in your own shell, with the same limits",
|
|
24
|
+
options: Type.Object({
|
|
25
|
+
run: Type.String({ description: 'the command line, run by bash from your workspace, e.g. "df -h /" or "git -C ~/dev/app rev-parse origin/main". One that only looks: it runs unattended, again and again' }),
|
|
26
|
+
every: Type.Optional(Type.String({ description: 'how often to run it, e.g. "1m", "1h". Default "1m", at least "5s"' })),
|
|
27
|
+
match: Type.Optional(Type.String({ description: 'watch only what this regular expression finds in what it prints (its first group, if it has one), e.g. "([0-9]+)% used". Without it everything it prints counts' })),
|
|
28
|
+
timeout: Type.Optional(Type.String({ description: 'how long one run may take before it is stopped. Default "30s"' })),
|
|
29
|
+
}, { additionalProperties: false }),
|
|
30
|
+
plan(o, setup) {
|
|
31
|
+
const run = String(o.run ?? "").trim();
|
|
32
|
+
if (!run)
|
|
33
|
+
throw new Error(`needs "run": the command line`);
|
|
34
|
+
if (run.length > 2000)
|
|
35
|
+
throw new Error("that command line is too long: put it into a script in your workspace and run that");
|
|
36
|
+
const every = o.every ?? "1m";
|
|
37
|
+
if (parseDuration(every) < MIN)
|
|
38
|
+
throw new Error('"every" must be at least "5s"');
|
|
39
|
+
if (o.timeout && parseDuration(o.timeout) < 1000)
|
|
40
|
+
throw new Error('"timeout" must be at least "1s"');
|
|
41
|
+
if (o.match)
|
|
42
|
+
pattern(o.match);
|
|
43
|
+
return {
|
|
44
|
+
options: { ...o, run },
|
|
45
|
+
does: [`run this command every ${every}, unattended, with the rights of ${setup.agent}'s shell (its sandbox): ${run}`, `report when ${o.match ? `what /${o.match}/ finds in what it prints` : "what it prints"} or its exit code changes`],
|
|
46
|
+
opens: true,
|
|
47
|
+
tell: "Its first run is the starting point and reports nothing; from then on only a change does. The sources tool shows how the last run went.",
|
|
48
|
+
};
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
export default function command(source) {
|
|
52
|
+
const o = source.options;
|
|
53
|
+
const run = typeof o.run === "string" ? o.run.trim() : "";
|
|
54
|
+
if (!run)
|
|
55
|
+
throw new Error(`needs "run": the command line`);
|
|
56
|
+
const re = o.match ? pattern(String(o.match)) : undefined;
|
|
57
|
+
const timeoutMs = parseDuration(o.timeout ?? "30s");
|
|
58
|
+
source.poll(o.every ?? "1m", async (signal) => {
|
|
59
|
+
const { code, output } = await source.exec(run, { timeoutMs, signal });
|
|
60
|
+
const text = output.replace(/\r\n/g, "\n").trimEnd().slice(0, 200_000);
|
|
61
|
+
if (re) {
|
|
62
|
+
const value = found(re, text);
|
|
63
|
+
source.status(`exit ${code}, ${value === null ? "found nothing with the pattern" : `now: ${harmless(value)}`}`);
|
|
64
|
+
const was = source.changed({ code, value });
|
|
65
|
+
if (was)
|
|
66
|
+
source.emit("changed", { command: clip(run, 300), code, value, previous: shown(was.previous.value), ...(was.previous.code !== code ? { previousCode: was.previous.code } : {}) });
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
source.status(`exit ${code}, ${text ? text.split("\n").length : 0} lines`);
|
|
70
|
+
const was = source.changed({ code, text });
|
|
71
|
+
if (!was)
|
|
72
|
+
return;
|
|
73
|
+
const diff = lineDiff(String(was.previous.text ?? ""), text);
|
|
74
|
+
if (diff.added.length || diff.removed.length || was.previous.code !== code)
|
|
75
|
+
source.emit("changed", { command: clip(run, 300), code, ...(was.previous.code !== code ? { previousCode: was.previous.code } : {}), ...diff });
|
|
76
|
+
});
|
|
77
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SourceType } from "../source-types.ts";
|
|
2
|
+
import type { SourceApi } from "../sources.ts";
|
|
3
|
+
/** A file name mask ("*.pdf", "*.{png,jpg}", "a?.txt, b*.md") as a test for a file's name. Capitals do not matter. */
|
|
4
|
+
export declare function maskTest(mask: string): RegExp;
|
|
5
|
+
export declare const type: SourceType;
|
|
6
|
+
export default function folder(source: SourceApi): void;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { Type } from "typebox";
|
|
4
|
+
import { isInside, realish } from "../security.js";
|
|
5
|
+
import { parseDuration, resolvePath } from "../util.js";
|
|
6
|
+
/**
|
|
7
|
+
* A folder: reports every new or changed file once it has stopped growing. For things that drop
|
|
8
|
+
* files: a scanner, a download folder, an export from another program.
|
|
9
|
+
*
|
|
10
|
+
* options: { dir, mask?, pattern?, ignore?, settle?, recursive? }
|
|
11
|
+
* dir the folder to watch (~ and paths relative to the configuration file work)
|
|
12
|
+
* mask only files whose name matches, as in a shell: "*.pdf", "*.{png,jpg}", "report-??.csv";
|
|
13
|
+
* several separated by commas
|
|
14
|
+
* pattern only files whose full path matches this regular expression, e.g. "/src/.*\\.ts$"
|
|
15
|
+
* ignore never files whose path matches this one. Default: version control, dependencies and
|
|
16
|
+
* editor leftovers (.git, node_modules, .DS_Store, files ending in ~ or .swp, and such)
|
|
17
|
+
* settle how long a file must stay unchanged before it counts as complete (default "1.5s")
|
|
18
|
+
* recursive also watch folders inside it (default true)
|
|
19
|
+
*
|
|
20
|
+
* event "added": { path, file, size, images? } images is set for jpg, png, webp and gif under 10 MB
|
|
21
|
+
*/
|
|
22
|
+
const TYPES = { ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", ".gif": "image/gif" };
|
|
23
|
+
/** A file name mask ("*.pdf", "*.{png,jpg}", "a?.txt, b*.md") as a test for a file's name. Capitals do not matter. */
|
|
24
|
+
export function maskTest(mask) {
|
|
25
|
+
const one = (glob) => glob
|
|
26
|
+
.trim()
|
|
27
|
+
.replace(/[.+^$()|[\]\\]/g, "\\$&")
|
|
28
|
+
.replace(/\*/g, "[^/]*")
|
|
29
|
+
.replace(/\?/g, "[^/]")
|
|
30
|
+
.replace(/\{([^{}]*)\}/g, (_m, list) => `(?:${list.split(",").map((x) => x.trim()).join("|")})`);
|
|
31
|
+
// Commas separate masks, except inside braces.
|
|
32
|
+
const parts = mask.split(/,(?![^{]*\})/).map(one).filter(Boolean);
|
|
33
|
+
return new RegExp(`^(?:${parts.join("|")})$`, "i");
|
|
34
|
+
}
|
|
35
|
+
export const type = {
|
|
36
|
+
summary: "a folder: reports every new or changed file in it. For a project folder, what a scanner saves, downloads, an export from another program",
|
|
37
|
+
options: Type.Object({
|
|
38
|
+
dir: Type.String({ description: "the folder to watch, absolute or starting with ~" }),
|
|
39
|
+
mask: Type.Optional(Type.String({ description: 'only files whose name matches, as in a shell: "*.pdf", "*.{png,jpg}"; several separated by commas' })),
|
|
40
|
+
pattern: Type.Optional(Type.String({ description: 'only files whose full path matches this regular expression, for what a mask cannot say, e.g. "/src/.*\\.ts$"' })),
|
|
41
|
+
ignore: Type.Optional(Type.String({ description: "never files whose path matches this regular expression. Default: version control, dependencies and editor leftovers (.git, node_modules, .DS_Store and such)" })),
|
|
42
|
+
recursive: Type.Optional(Type.Boolean({ description: "also watch the folders inside it. Default true" })),
|
|
43
|
+
settle: Type.Optional(Type.String({ description: 'how long a file must stay unchanged before it counts as complete. Default "1.5s"' })),
|
|
44
|
+
}, { additionalProperties: false }),
|
|
45
|
+
plan(o, setup) {
|
|
46
|
+
const dir = setup.resolve(o.dir);
|
|
47
|
+
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory())
|
|
48
|
+
throw new Error(`${dir} is not a folder that exists`);
|
|
49
|
+
// Its own workspace is nothing new; a folder elsewhere is, and reading its files comes with watching it.
|
|
50
|
+
const own = isInside(realish(setup.workspace), dir);
|
|
51
|
+
// The workspace usually lies inside pine's home; everything else in there is pine's own.
|
|
52
|
+
if (!own && (isInside(realish(setup.home), dir) || isInside(dir, realish(setup.home))))
|
|
53
|
+
throw new Error("pine's own folder cannot be watched");
|
|
54
|
+
for (const key of ["pattern", "ignore"])
|
|
55
|
+
if (o[key])
|
|
56
|
+
new RegExp(o[key]);
|
|
57
|
+
if (o.mask)
|
|
58
|
+
maskTest(o.mask);
|
|
59
|
+
if (o.settle)
|
|
60
|
+
parseDuration(o.settle);
|
|
61
|
+
const only = [o.mask ? `files named ${o.mask}` : "", o.pattern ? `paths matching ${o.pattern}` : ""].filter(Boolean).join(", ");
|
|
62
|
+
return { options: { ...o, dir }, does: [`watch the folder ${dir}${only ? ` (${only})` : ""} for new and changed files`, ...(own ? [] : [`let ${setup.agent} read the files in that folder`])], opens: !own, read: own ? undefined : dir };
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
export default function folder(source) {
|
|
66
|
+
const o = source.options;
|
|
67
|
+
if (typeof o.dir !== "string" || !o.dir)
|
|
68
|
+
throw new Error(`needs "dir": the folder to watch`);
|
|
69
|
+
const dir = fs.realpathSync(resolvePath(o.dir, source.base));
|
|
70
|
+
if (!fs.statSync(dir).isDirectory())
|
|
71
|
+
throw new Error(`${dir} is not a folder`);
|
|
72
|
+
const pattern = o.pattern ? new RegExp(String(o.pattern), "i") : undefined;
|
|
73
|
+
const mask = typeof o.mask === "string" && o.mask.trim() ? maskTest(o.mask) : undefined;
|
|
74
|
+
// What changes all the time in a project folder without anyone having done anything worth a turn.
|
|
75
|
+
const ignore = new RegExp(typeof o.ignore === "string" ? o.ignore : "(^|/)(\\.git|\\.hg|\\.svn|node_modules|\\.build|DerivedData|__pycache__|\\.venv|\\.DS_Store)(/|$)|(~|\\.swp|\\.tmp|\\.part|\\.crdownload)$", "i");
|
|
76
|
+
const settleMs = parseDuration(o.settle ?? "1.5s");
|
|
77
|
+
const pending = new Map();
|
|
78
|
+
const seen = new Map();
|
|
79
|
+
source.media(dir);
|
|
80
|
+
const check = (file, lastSize) => {
|
|
81
|
+
pending.delete(file);
|
|
82
|
+
let stat;
|
|
83
|
+
try {
|
|
84
|
+
stat = fs.statSync(file);
|
|
85
|
+
// A link that leads out of the folder is not a file of this folder.
|
|
86
|
+
if (!stat.isFile() || !fs.realpathSync(file).startsWith(dir + path.sep))
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
if (stat.size !== lastSize || stat.size === 0)
|
|
93
|
+
return void pending.set(file, setTimeout(() => check(file, stat.size), settleMs));
|
|
94
|
+
if (seen.get(file) === stat.mtimeMs)
|
|
95
|
+
return;
|
|
96
|
+
seen.set(file, stat.mtimeMs);
|
|
97
|
+
if (seen.size > 5000)
|
|
98
|
+
for (const k of [...seen.keys()].slice(0, 2500))
|
|
99
|
+
seen.delete(k);
|
|
100
|
+
const mimeType = TYPES[path.extname(file).toLowerCase()];
|
|
101
|
+
source.emit("added", { path: file, file: path.relative(dir, file), size: stat.size, images: mimeType && stat.size < 10 * 1024 * 1024 ? [{ path: file, mimeType }] : undefined });
|
|
102
|
+
};
|
|
103
|
+
const watcher = fs.watch(dir, { recursive: o.recursive !== false }, (_type, name) => {
|
|
104
|
+
if (!name)
|
|
105
|
+
return;
|
|
106
|
+
const file = path.join(dir, String(name));
|
|
107
|
+
if ((pattern && !pattern.test(file)) || (mask && !mask.test(path.basename(file))) || ignore.test(String(name)) || pending.has(file))
|
|
108
|
+
return;
|
|
109
|
+
pending.set(file, setTimeout(() => check(file, -1), 200));
|
|
110
|
+
});
|
|
111
|
+
source.onStop(() => {
|
|
112
|
+
watcher.close();
|
|
113
|
+
pending.forEach((t) => clearTimeout(t));
|
|
114
|
+
});
|
|
115
|
+
source.log.info(`watching ${dir}`);
|
|
116
|
+
}
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import net from "node:net";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { timingSafeEqual } from "../security.js";
|
|
5
|
+
import { Type } from "typebox";
|
|
6
|
+
import { errText } from "../util.js";
|
|
7
|
+
/**
|
|
8
|
+
* An FTP drop: a small FTP server that accepts uploads and nothing else. For cameras, doorbells,
|
|
9
|
+
* scanners and other devices whose only way of sending a file is "upload to FTP server".
|
|
10
|
+
*
|
|
11
|
+
* options: { port, user, password, host?, allow?, pasvPorts?, maxFileMb?, keepDays? }
|
|
12
|
+
* port, user, password what you enter in the device. The password must have 8 characters or more.
|
|
13
|
+
* host address to listen on. Default "0.0.0.0": devices on your network have to reach it.
|
|
14
|
+
* allow only these client addresses, as prefixes: ["192.168.1.40", "192.168.2."]
|
|
15
|
+
* pasvPorts [from, to]: ports for the data connections, if a firewall needs to know them
|
|
16
|
+
* maxFileMb largest file accepted (default 50)
|
|
17
|
+
* keepDays received files are deleted after this many days (default 7)
|
|
18
|
+
*
|
|
19
|
+
* event "upload": { file, path, size, from, images? } file is the name the device gave it
|
|
20
|
+
*
|
|
21
|
+
* It speaks plain FTP, as these devices do: the password crosses your network unencrypted. It can
|
|
22
|
+
* only receive. Nothing can be listed, read, changed or deleted through it, and every file lands
|
|
23
|
+
* under a name of pine's choosing inside the source's own folder.
|
|
24
|
+
*/
|
|
25
|
+
const TYPES = { ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", ".gif": "image/gif" };
|
|
26
|
+
export const type = {
|
|
27
|
+
summary: "an FTP drop: a small server that accepts uploads and nothing else. For cameras, doorbells and scanners whose only way of sending a file is \"upload to FTP server\"",
|
|
28
|
+
options: Type.Object({
|
|
29
|
+
port: Type.Number({ description: "port to listen on, e.g. 2121" }),
|
|
30
|
+
user: Type.Optional(Type.String({ description: 'user name the device logs in with. Default "device"' })),
|
|
31
|
+
password: Type.Optional(Type.String({ description: "password the device logs in with, 8 characters or more. Leave it out and pine makes one", secret: true })),
|
|
32
|
+
host: Type.Optional(Type.String({ description: 'address to listen on. Default "0.0.0.0", so that devices on the network reach it' })),
|
|
33
|
+
allow: Type.Optional(Type.Array(Type.String(), { description: 'only these client addresses, as prefixes: ["192.168.1.40", "192.168.2."]' })),
|
|
34
|
+
pasvPorts: Type.Optional(Type.Array(Type.Number(), { description: "[from, to]: ports for the data connections, if a firewall needs to know them" })),
|
|
35
|
+
maxFileMb: Type.Optional(Type.Number({ description: "largest file accepted, in MB. Default 50" })),
|
|
36
|
+
keepDays: Type.Optional(Type.Number({ description: "received files are deleted after this many days. Default 7" })),
|
|
37
|
+
}, { additionalProperties: false }),
|
|
38
|
+
plan(o, setup) {
|
|
39
|
+
if (!Number.isInteger(o.port) || o.port < 1 || o.port > 65535)
|
|
40
|
+
throw new Error(`"port" must be a port number, for example 2121`);
|
|
41
|
+
if (o.password !== undefined && String(o.password).length < 8)
|
|
42
|
+
throw new Error("the password must have 8 characters or more");
|
|
43
|
+
if (o.pasvPorts && o.pasvPorts.length !== 2)
|
|
44
|
+
throw new Error(`"pasvPorts" is two numbers: [from, to]`);
|
|
45
|
+
const user = o.user ?? "device";
|
|
46
|
+
return {
|
|
47
|
+
options: { ...o, user, password: o.password ?? setup.random(9) },
|
|
48
|
+
does: [`open an upload-only FTP server on port ${o.port} of this computer for the user "${user}"${o.password ? "" : " with a password made now"}${o.allow?.length ? `, for the addresses ${o.allow.join(", ")}` : ""}`],
|
|
49
|
+
opens: true,
|
|
50
|
+
tell: `The device uploads to port ${o.port} as user "${user}". The password is in pine's .env file as ${setup.envName("password")}, where your owner reads it.`,
|
|
51
|
+
};
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
export default async function ftp(source) {
|
|
55
|
+
const o = source.options;
|
|
56
|
+
const port = Number(o.port);
|
|
57
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535)
|
|
58
|
+
throw new Error(`needs "port" (for example 2121)`);
|
|
59
|
+
const user = String(o.user ?? "");
|
|
60
|
+
const password = String(o.password ?? "");
|
|
61
|
+
if (!user || password.length < 8)
|
|
62
|
+
throw new Error(`needs "user" and a "password" of 8 characters or more: without them anyone on the network could hand files to your agents`);
|
|
63
|
+
source.secret(password);
|
|
64
|
+
const host = String(o.host ?? "0.0.0.0");
|
|
65
|
+
const allow = (Array.isArray(o.allow) ? o.allow : []).map(String);
|
|
66
|
+
const range = Array.isArray(o.pasvPorts) && o.pasvPorts.length === 2 ? [Number(o.pasvPorts[0]), Number(o.pasvPorts[1])] : undefined;
|
|
67
|
+
const maxBytes = Number(o.maxFileMb ?? 50) * 1024 * 1024;
|
|
68
|
+
const keepMs = Number(o.keepDays ?? 7) * 86_400_000;
|
|
69
|
+
const dir = path.join(source.dataDir, "files");
|
|
70
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
71
|
+
source.media(dir);
|
|
72
|
+
const prune = () => {
|
|
73
|
+
for (const name of fs.readdirSync(dir)) {
|
|
74
|
+
const file = path.join(dir, name);
|
|
75
|
+
try {
|
|
76
|
+
if (Date.now() - fs.statSync(file).mtimeMs > keepMs)
|
|
77
|
+
fs.rmSync(file, { force: true });
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
// gone already
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
prune();
|
|
85
|
+
const pruner = setInterval(prune, 6 * 3_600_000);
|
|
86
|
+
pruner.unref?.();
|
|
87
|
+
const plain = (addr) => (addr ?? "").replace(/^::ffff:/, "");
|
|
88
|
+
const open = new Set();
|
|
89
|
+
let counter = 0;
|
|
90
|
+
const server = net.createServer((control) => {
|
|
91
|
+
const remote = plain(control.remoteAddress);
|
|
92
|
+
if (open.size >= 20 || (allow.length && !allow.some((a) => remote === a || (a.endsWith(".") && remote.startsWith(a)))))
|
|
93
|
+
return void control.destroy();
|
|
94
|
+
open.add(control);
|
|
95
|
+
control.setTimeout(120_000, () => control.destroy());
|
|
96
|
+
control.setEncoding("utf8");
|
|
97
|
+
let name = "";
|
|
98
|
+
let authed = false;
|
|
99
|
+
let failures = 0;
|
|
100
|
+
let cwd = "/";
|
|
101
|
+
let passive;
|
|
102
|
+
let data;
|
|
103
|
+
const say = (line) => control.writable && control.write(`${line}\r\n`);
|
|
104
|
+
const closePassive = () => {
|
|
105
|
+
passive?.close();
|
|
106
|
+
passive = undefined;
|
|
107
|
+
data?.then((s) => s.destroy()).catch(() => { });
|
|
108
|
+
data = undefined;
|
|
109
|
+
};
|
|
110
|
+
control.on("close", () => {
|
|
111
|
+
open.delete(control);
|
|
112
|
+
closePassive();
|
|
113
|
+
});
|
|
114
|
+
control.on("error", () => { });
|
|
115
|
+
/** Open a port for the next transfer. Only the device on the control connection may connect to it. */
|
|
116
|
+
const listen = async () => {
|
|
117
|
+
closePassive();
|
|
118
|
+
const s = net.createServer();
|
|
119
|
+
passive = s;
|
|
120
|
+
data = new Promise((resolve, reject) => {
|
|
121
|
+
const timer = setTimeout(() => reject(new Error("no data connection")), 20_000);
|
|
122
|
+
s.on("connection", (sock) => {
|
|
123
|
+
if (plain(sock.remoteAddress) !== remote)
|
|
124
|
+
return void sock.destroy();
|
|
125
|
+
clearTimeout(timer);
|
|
126
|
+
s.close();
|
|
127
|
+
resolve(sock);
|
|
128
|
+
});
|
|
129
|
+
s.on("close", () => clearTimeout(timer));
|
|
130
|
+
});
|
|
131
|
+
data.catch(() => { });
|
|
132
|
+
const tryPort = (p) => new Promise((resolve, reject) => {
|
|
133
|
+
s.once("error", reject);
|
|
134
|
+
s.listen(p, host === "0.0.0.0" ? plain(control.localAddress) : host, () => (s.removeListener("error", reject), resolve(s.address().port)));
|
|
135
|
+
});
|
|
136
|
+
if (!range)
|
|
137
|
+
return tryPort(0);
|
|
138
|
+
for (let p = range[0]; p <= range[1]; p++) {
|
|
139
|
+
try {
|
|
140
|
+
return await tryPort(p);
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
// taken; try the next one
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
throw new Error("no free data port");
|
|
147
|
+
};
|
|
148
|
+
const receive = async (given) => {
|
|
149
|
+
if (!data)
|
|
150
|
+
return say("425 Use PASV first");
|
|
151
|
+
const wanted = path.posix.join(cwd, given);
|
|
152
|
+
const clean = path.basename(given).replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "").slice(-80) || "file";
|
|
153
|
+
const target = path.join(dir, `${Date.now().toString(36)}${(counter++).toString(36)}-${clean}`);
|
|
154
|
+
say("150 Send it");
|
|
155
|
+
let sock;
|
|
156
|
+
try {
|
|
157
|
+
sock = await data;
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
return say("425 No data connection");
|
|
161
|
+
}
|
|
162
|
+
data = undefined;
|
|
163
|
+
let size = 0;
|
|
164
|
+
let tooBig = false;
|
|
165
|
+
// "wx": the name is new; nothing that exists is ever written over.
|
|
166
|
+
const out = fs.createWriteStream(target, { flags: "wx", mode: 0o600 });
|
|
167
|
+
sock.on("data", (chunk) => {
|
|
168
|
+
size += chunk.length;
|
|
169
|
+
if (size > maxBytes) {
|
|
170
|
+
tooBig = true;
|
|
171
|
+
sock.destroy();
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
sock.pipe(out);
|
|
175
|
+
await new Promise((resolve) => {
|
|
176
|
+
out.on("close", resolve);
|
|
177
|
+
out.on("error", () => resolve());
|
|
178
|
+
// A connection that is cut (by the device, or here because the file got too large) never
|
|
179
|
+
// says "end"; the file is closed all the same, so the transfer gets its answer.
|
|
180
|
+
sock.on("close", () => out.end());
|
|
181
|
+
sock.on("error", () => out.end());
|
|
182
|
+
});
|
|
183
|
+
if (tooBig || size === 0) {
|
|
184
|
+
fs.rmSync(target, { force: true });
|
|
185
|
+
return say(tooBig ? "552 File too large" : "451 Nothing received");
|
|
186
|
+
}
|
|
187
|
+
say("226 Stored");
|
|
188
|
+
const mimeType = TYPES[path.extname(clean).toLowerCase()];
|
|
189
|
+
try {
|
|
190
|
+
source.emit("upload", { file: wanted, path: target, size, from: remote, images: mimeType && size < 10 * 1024 * 1024 ? [{ path: target, mimeType }] : undefined });
|
|
191
|
+
}
|
|
192
|
+
catch (e) {
|
|
193
|
+
source.log.warn("could not report an upload", errText(e));
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
const handle = async (line) => {
|
|
197
|
+
const [verb, ...rest] = line.trim().split(" ");
|
|
198
|
+
const arg = rest.join(" ");
|
|
199
|
+
const cmd = verb.toUpperCase();
|
|
200
|
+
if (cmd === "QUIT")
|
|
201
|
+
return void (say("221 Bye"), control.end());
|
|
202
|
+
if (cmd === "USER")
|
|
203
|
+
return void ((name = arg), say("331 Password"));
|
|
204
|
+
if (cmd === "PASS") {
|
|
205
|
+
authed = timingSafeEqual(name, user) && timingSafeEqual(arg, password);
|
|
206
|
+
if (authed)
|
|
207
|
+
return say("230 Welcome");
|
|
208
|
+
if (++failures >= 3)
|
|
209
|
+
return void (say("530 Bye"), control.end());
|
|
210
|
+
return say("530 Login incorrect");
|
|
211
|
+
}
|
|
212
|
+
if (cmd === "SYST")
|
|
213
|
+
return say("215 UNIX Type: L8");
|
|
214
|
+
if (cmd === "FEAT")
|
|
215
|
+
return say("211-Features:\r\n PASV\r\n EPSV\r\n UTF8\r\n211 End");
|
|
216
|
+
if (!authed)
|
|
217
|
+
return say("530 Log in first");
|
|
218
|
+
switch (cmd) {
|
|
219
|
+
case "OPTS":
|
|
220
|
+
case "TYPE":
|
|
221
|
+
case "MODE":
|
|
222
|
+
case "STRU":
|
|
223
|
+
case "NOOP":
|
|
224
|
+
case "ALLO":
|
|
225
|
+
return say("200 OK");
|
|
226
|
+
case "PWD":
|
|
227
|
+
case "XPWD":
|
|
228
|
+
return say(`257 "${cwd}"`);
|
|
229
|
+
case "CWD":
|
|
230
|
+
case "XCWD":
|
|
231
|
+
cwd = path.posix.resolve(cwd, arg || "/");
|
|
232
|
+
return say("250 OK");
|
|
233
|
+
case "CDUP":
|
|
234
|
+
cwd = path.posix.resolve(cwd, "..");
|
|
235
|
+
return say("250 OK");
|
|
236
|
+
// Folders exist only as part of the name a file is reported under.
|
|
237
|
+
case "MKD":
|
|
238
|
+
case "XMKD":
|
|
239
|
+
return say(`257 "${path.posix.resolve(cwd, arg)}"`);
|
|
240
|
+
case "PASV": {
|
|
241
|
+
const p = await listen();
|
|
242
|
+
const ip = plain(control.localAddress).split(".");
|
|
243
|
+
if (ip.length !== 4)
|
|
244
|
+
return say("425 Use EPSV");
|
|
245
|
+
return say(`227 Entering Passive Mode (${ip.join(",")},${p >> 8},${p & 255})`);
|
|
246
|
+
}
|
|
247
|
+
case "EPSV":
|
|
248
|
+
return say(`229 Entering Extended Passive Mode (|||${await listen()}|)`);
|
|
249
|
+
case "STOR":
|
|
250
|
+
case "APPE":
|
|
251
|
+
case "STOU":
|
|
252
|
+
return receive(arg || "file");
|
|
253
|
+
case "LIST":
|
|
254
|
+
case "NLST":
|
|
255
|
+
case "MLSD": {
|
|
256
|
+
// Always empty: this server gives nothing back.
|
|
257
|
+
if (!data)
|
|
258
|
+
return say("425 Use PASV first");
|
|
259
|
+
say("150 Here it comes");
|
|
260
|
+
const sock = await data.catch(() => undefined);
|
|
261
|
+
data = undefined;
|
|
262
|
+
sock?.end();
|
|
263
|
+
return say("226 Done");
|
|
264
|
+
}
|
|
265
|
+
// Some devices upload under a temporary name and rename afterwards. The file is already reported; say yes.
|
|
266
|
+
case "RNFR":
|
|
267
|
+
return say("350 Ready");
|
|
268
|
+
case "RNTO":
|
|
269
|
+
case "DELE":
|
|
270
|
+
case "RMD":
|
|
271
|
+
case "SITE":
|
|
272
|
+
return say("250 OK");
|
|
273
|
+
case "SIZE":
|
|
274
|
+
case "MDTM":
|
|
275
|
+
case "RETR":
|
|
276
|
+
return say("550 Not available");
|
|
277
|
+
default:
|
|
278
|
+
return say("502 Not implemented");
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
let buffer = "";
|
|
282
|
+
let busy = Promise.resolve();
|
|
283
|
+
control.on("data", (chunk) => {
|
|
284
|
+
buffer += chunk;
|
|
285
|
+
if (buffer.length > 8192)
|
|
286
|
+
return void control.destroy();
|
|
287
|
+
let i;
|
|
288
|
+
while ((i = buffer.indexOf("\n")) >= 0) {
|
|
289
|
+
const line = buffer.slice(0, i).replace(/\r$/, "");
|
|
290
|
+
buffer = buffer.slice(i + 1);
|
|
291
|
+
busy = busy.then(() => handle(line)).catch((e) => (source.log.debug("ftp command failed", errText(e)), say("451 Failed")));
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
say("220 pine file drop");
|
|
295
|
+
});
|
|
296
|
+
await new Promise((resolve, reject) => {
|
|
297
|
+
server.once("error", reject);
|
|
298
|
+
server.listen(port, host, () => (server.removeListener("error", reject), resolve()));
|
|
299
|
+
});
|
|
300
|
+
source.onStop(() => new Promise((resolve) => {
|
|
301
|
+
clearInterval(pruner);
|
|
302
|
+
for (const s of open)
|
|
303
|
+
s.destroy();
|
|
304
|
+
server.close(() => resolve());
|
|
305
|
+
}));
|
|
306
|
+
source.log.info(`accepting uploads on ${host}:${port}`);
|
|
307
|
+
}
|