dsh-remote 0.7.4 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -16
- package/lib/client.js +408 -152
- package/lib/credential.js +107 -0
- package/lib/errors.js +78 -0
- package/lib/forwards.js +219 -0
- package/lib/hostkey.js +48 -0
- package/lib/ignore.js +123 -0
- package/lib/index.js +1458 -841
- package/lib/paths.js +169 -0
- package/lib/search.js +127 -0
- package/lib/sshconfig.js +57 -0
- package/lib/sync.js +270 -0
- package/lib/tasks.js +95 -0
- package/lib/update.js +196 -0
- package/package.json +7 -3
package/lib/paths.js
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// dsh-remote — pure path / shell helpers (no fs, no ssh).
|
|
2
|
+
// Split out of lib/index.js so they are unit-testable without loading the
|
|
3
|
+
// plugin. Both POSIX (`/a/b`) and Windows (`D:\Code`, `\\server\share`) forms
|
|
4
|
+
// are supported everywhere.
|
|
5
|
+
|
|
6
|
+
/** Single-quote one shell argument verbatim. */
|
|
7
|
+
export function shq(s) {
|
|
8
|
+
return "'" + String(s).replace(/'/g, "'\\''") + "'"
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Normalize a remote path into a clean absolute path, supporting BOTH POSIX
|
|
13
|
+
* (`/home/user`) and Windows (`D:\Code`, `C:/Users/x`, `\\server\share`) forms.
|
|
14
|
+
*
|
|
15
|
+
* - POSIX paths keep `/` separators and a leading `/`.
|
|
16
|
+
* - Windows drive paths keep their `X:` prefix and are returned with `\`
|
|
17
|
+
* separators (the native form on a Windows remote); a leading `/` is NOT
|
|
18
|
+
* added, so `D:\Code` stays `D:\Code` instead of becoming `/D:\Code`.
|
|
19
|
+
* - UNC paths (`\\server\share\…`) keep the `\\server\share` prefix.
|
|
20
|
+
* - `.` / `..` segments are collapsed in both separator styles.
|
|
21
|
+
*/
|
|
22
|
+
export function normalizeRemotePath(p) {
|
|
23
|
+
const s = String(p)
|
|
24
|
+
// Windows UNC: \\server\share\…
|
|
25
|
+
const unc = s.match(/^(\\\\[^\\]+(?:\\[^\\]+)?)(?:\\|$)(.*)$/s)
|
|
26
|
+
if (unc) {
|
|
27
|
+
const [prefix, rest] = [unc[1], unc[2]]
|
|
28
|
+
const parts = []
|
|
29
|
+
for (const seg of rest.split(/[\\/]+/)) {
|
|
30
|
+
if (seg === '' || seg === '.') continue
|
|
31
|
+
if (seg === '..') { parts.pop(); continue }
|
|
32
|
+
parts.push(seg)
|
|
33
|
+
}
|
|
34
|
+
return parts.length ? prefix + '\\' + parts.join('\\') : prefix
|
|
35
|
+
}
|
|
36
|
+
// Windows drive: X:\… or X:/…
|
|
37
|
+
const drive = s.match(/^([a-zA-Z]:)(?:[\\/]|$)(.*)$/s)
|
|
38
|
+
if (drive) {
|
|
39
|
+
const [prefix, rest] = [drive[1], drive[2]]
|
|
40
|
+
const parts = []
|
|
41
|
+
for (const seg of rest.split(/[\\/]+/)) {
|
|
42
|
+
if (seg === '' || seg === '.') continue
|
|
43
|
+
if (seg === '..') { parts.pop(); continue }
|
|
44
|
+
parts.push(seg)
|
|
45
|
+
}
|
|
46
|
+
return prefix + '\\' + parts.join('\\')
|
|
47
|
+
}
|
|
48
|
+
// POSIX: /a/b/c
|
|
49
|
+
const parts = []
|
|
50
|
+
for (const seg of s.split('/')) {
|
|
51
|
+
if (seg === '' || seg === '.') continue
|
|
52
|
+
if (seg === '..') {
|
|
53
|
+
parts.pop()
|
|
54
|
+
continue
|
|
55
|
+
}
|
|
56
|
+
parts.push(seg)
|
|
57
|
+
}
|
|
58
|
+
return '/' + parts.join('/')
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Join a remote dir + entry name, honoring the dir's own separator style. */
|
|
62
|
+
export function joinRemotePath(dir, name) {
|
|
63
|
+
const d = String(dir)
|
|
64
|
+
if (!d) return String(name)
|
|
65
|
+
if (d.endsWith('/') || d.endsWith('\\')) return d + String(name)
|
|
66
|
+
return d + (d.includes('\\') ? '\\' : '/') + String(name)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Parent dir of a remote absolute path (string-level; POSIX or Windows). */
|
|
70
|
+
export function remoteDirname(p) {
|
|
71
|
+
const norm = normalizeRemotePath(p)
|
|
72
|
+
// Windows drive/UNC root: D:\ or \\server\share
|
|
73
|
+
if (norm === '/' || /^[a-zA-Z]:\\?$/.test(norm) || /^\\\\[^\\]+\\[^\\]+$/.test(norm)) {
|
|
74
|
+
// Ensure a drive root ends with a backslash: D: → D:\
|
|
75
|
+
const m = norm.match(/^([a-zA-Z]:)$/)
|
|
76
|
+
return m ? m[1] + '\\' : norm
|
|
77
|
+
}
|
|
78
|
+
const sep = norm.includes('\\') ? '\\' : '/'
|
|
79
|
+
const idx = norm.lastIndexOf(sep)
|
|
80
|
+
if (idx <= 0) {
|
|
81
|
+
// Windows drive with a child (D:\Code) → D:\
|
|
82
|
+
const m = norm.match(/^([a-zA-Z]:)\\/)
|
|
83
|
+
return m ? m[1] + '\\' : sep === '/' && norm.startsWith('/') ? '/' : norm
|
|
84
|
+
}
|
|
85
|
+
// A parent that is a Windows drive root also needs the trailing backslash.
|
|
86
|
+
const parent = norm.slice(0, idx)
|
|
87
|
+
if (sep === '\\' && /^[a-zA-Z]:$/.test(parent)) return parent + '\\'
|
|
88
|
+
return parent
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Create every level of a remote dir via SFTP (true mkdir -p: parents AND the
|
|
92
|
+
* target itself), working for both POSIX (`/a/b`) and Windows (`D:\a\b`)
|
|
93
|
+
* paths. Best effort: an existing or permission-denied level is skipped.
|
|
94
|
+
* FILE-oriented callers must pass the parent dir (remoteDirname(filePath)). */
|
|
95
|
+
export async function mkdirRemoteDirs(sftp, dir) {
|
|
96
|
+
const target = normalizeRemotePath(dir)
|
|
97
|
+
const isWin = target.includes('\\')
|
|
98
|
+
// Split off a Windows drive prefix (D:) so it is not treated as a segment.
|
|
99
|
+
const drive = isWin ? (target.match(/^([a-zA-Z]:)[\\/]?(.*)$/s) || null) : null
|
|
100
|
+
const segs = drive ? drive[2].split(/[\\/]/).filter(Boolean) : target.split(/[\\/]/).filter(Boolean)
|
|
101
|
+
let cur = isWin ? (drive ? drive[1] + '\\' : '') : ''
|
|
102
|
+
// Drive root (D:\) itself, best effort (usually exists).
|
|
103
|
+
if (isWin && cur) { try { await sftp.mkdir(cur) } catch { /* exists */ } }
|
|
104
|
+
for (const s of segs) {
|
|
105
|
+
if (cur) {
|
|
106
|
+
if (!cur.endsWith('\\') && !cur.endsWith('/')) cur += isWin ? '\\' : '/'
|
|
107
|
+
cur += s
|
|
108
|
+
} else {
|
|
109
|
+
cur = (isWin ? s + ':' : '/' + s)
|
|
110
|
+
}
|
|
111
|
+
try { await sftp.mkdir(cur) } catch { /* exists or no perms */ }
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Convert an internal remote path (POSIX `/a/b` or Windows `D:\Code`) into the
|
|
117
|
+
* path format the SFTP server understands.
|
|
118
|
+
*
|
|
119
|
+
* Windows OpenSSH's sftp-server accepts POSIX-style paths: a drive letter is
|
|
120
|
+
* written `/D:/…` (equivalent to `D:\…`). POSIX paths pass through unchanged.
|
|
121
|
+
* UNC paths are kept as-is (the server resolves `\\server\share` itself).
|
|
122
|
+
*/
|
|
123
|
+
export function toSftpPath(p) {
|
|
124
|
+
const norm = normalizeRemotePath(p)
|
|
125
|
+
const m = norm.match(/^([a-zA-Z]):\\(.*)$/s)
|
|
126
|
+
if (m) return '/' + m[1] + ':/' + m[2].replace(/\\/g, '/')
|
|
127
|
+
return norm
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Basename of a remote absolute path (POSIX or Windows). */
|
|
131
|
+
export function remotePathBase(p) {
|
|
132
|
+
const norm = normalizeRemotePath(p).replace(/[\\/]+$/, '')
|
|
133
|
+
const base = norm.split(/[\\/]/).pop()
|
|
134
|
+
// Windows drive root (D:\) has no meaningful basename
|
|
135
|
+
if (!base || /^[a-zA-Z]:$/.test(base)) return 'workspace'
|
|
136
|
+
return base
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Relative path of `p` under `root` (POSIX form, no leading slash), or null
|
|
140
|
+
* when `p` is not inside `root`. Both POSIX and Windows drive paths work
|
|
141
|
+
* (Windows is compared in its `/D:/…` POSIX-equivalent form). */
|
|
142
|
+
export function relPathUnder(root, p) {
|
|
143
|
+
const posix = (x) => {
|
|
144
|
+
let n = normalizeRemotePath(x).replace(/[\\/]+$/, '')
|
|
145
|
+
const m = n.match(/^([a-zA-Z]):(.*)$/s)
|
|
146
|
+
if (m) return '/' + m[1].toLowerCase() + ':/' + (m[2] ? m[2].replace(/\\/g, '/').replace(/^\/+/, '') : '')
|
|
147
|
+
return n.replace(/\\/g, '/')
|
|
148
|
+
}
|
|
149
|
+
const r = posix(root)
|
|
150
|
+
const n = posix(p)
|
|
151
|
+
if (n === r) return ''
|
|
152
|
+
const prefix = r === '/' ? '/' : r + '/'
|
|
153
|
+
if (n.startsWith(prefix)) return n.slice(prefix.length)
|
|
154
|
+
return null
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function truncate(s, max) {
|
|
158
|
+
if (s.length <= max) return s
|
|
159
|
+
return s.slice(0, max) + `\n…[truncated: ${s.length - max} more chars]`
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** 32-bit string hash rendered in base36 (short, collision-safe enough). */
|
|
163
|
+
export function shortHash(s) {
|
|
164
|
+
let h = 0
|
|
165
|
+
for (let i = 0; i < s.length; i++) {
|
|
166
|
+
h = ((h << 5) - h + s.charCodeAt(i)) | 0
|
|
167
|
+
}
|
|
168
|
+
return (h >>> 0).toString(36)
|
|
169
|
+
}
|
package/lib/search.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// dsh-remote — portable recursive search over SFTP.
|
|
2
|
+
// Replaces `find … -exec grep` (which needs a POSIX shell and dies on
|
|
3
|
+
// Windows remotes). Walks the remote tree over SFTP readdir/stat/readFile with
|
|
4
|
+
// the same bounded-parallelism + ignore rules as the mirror sync.
|
|
5
|
+
import { joinRemotePath } from './paths.js'
|
|
6
|
+
|
|
7
|
+
function mapLimit(items, limit, fn) {
|
|
8
|
+
const limitN = Math.max(1, Math.min(Math.floor(limit) || 1, items.length))
|
|
9
|
+
let next = 0
|
|
10
|
+
const worker = async () => {
|
|
11
|
+
while (next < items.length) {
|
|
12
|
+
const i = next++
|
|
13
|
+
try { await fn(items[i], i) } catch { /* per-item errors are the fn's concern */ }
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
const workers = []
|
|
17
|
+
for (let w = 0; w < limitN; w++) workers.push(worker())
|
|
18
|
+
return Promise.all(workers)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** fnmatch-lite for the `glob` filter (matches the entry BASENAME). */
|
|
22
|
+
export function matchGlob(name, glob) {
|
|
23
|
+
let re = ''
|
|
24
|
+
let i = 0
|
|
25
|
+
const g = String(glob || '')
|
|
26
|
+
while (i < g.length) {
|
|
27
|
+
const c = g[i]
|
|
28
|
+
if (c === '*') {
|
|
29
|
+
if (g[i + 1] === '*') { re += '.*'; i += 2; continue }
|
|
30
|
+
re += '[^/]*'; i++; continue
|
|
31
|
+
}
|
|
32
|
+
if (c === '?') { re += '[^/]'; i++; continue }
|
|
33
|
+
if ('\\^$+{}[]()|.'.includes(c)) re += '\\' + c
|
|
34
|
+
else re += c
|
|
35
|
+
i++
|
|
36
|
+
}
|
|
37
|
+
return new RegExp('^' + re + '$').test(String(name))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const BINARY_SNIFF = 8192
|
|
41
|
+
function looksBinary(buf) {
|
|
42
|
+
const n = Math.min(buf.length, BINARY_SNIFF)
|
|
43
|
+
for (let i = 0; i < n; i++) if (buf[i] === 0) return true
|
|
44
|
+
return false
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Recursively search a remote directory.
|
|
49
|
+
* @returns {Promise<{matches: Array<{path, line, text}>, scanned: number, truncated: boolean}>}
|
|
50
|
+
*/
|
|
51
|
+
export async function searchTree(sftp, rootDir, opts = {}) {
|
|
52
|
+
const {
|
|
53
|
+
regex, // RegExp (already compiled with ignoreCase where asked)
|
|
54
|
+
glob, // optional basename glob filter
|
|
55
|
+
contextLines = 0,
|
|
56
|
+
maxMatches = 500,
|
|
57
|
+
maxDepth = 12,
|
|
58
|
+
maxFiles = 20000,
|
|
59
|
+
maxScanBytes = 1024 * 1024,
|
|
60
|
+
isIgnored = () => false,
|
|
61
|
+
} = opts
|
|
62
|
+
|
|
63
|
+
const matches = []
|
|
64
|
+
let scanned = 0
|
|
65
|
+
let truncated = false
|
|
66
|
+
|
|
67
|
+
const walk = async (dir, depth) => {
|
|
68
|
+
if (truncated || scanned >= maxFiles) return
|
|
69
|
+
let entries = []
|
|
70
|
+
try { entries = (await sftp.readdir(dir)) || [] } catch { return }
|
|
71
|
+
const dirs = []
|
|
72
|
+
for (const e of entries) {
|
|
73
|
+
const name = String(e.filename)
|
|
74
|
+
if (name === '.' || name === '..') continue
|
|
75
|
+
const isDir = !!(e.attrs && e.attrs.isDirectory && e.attrs.isDirectory())
|
|
76
|
+
if (isDir) dirs.push(name)
|
|
77
|
+
}
|
|
78
|
+
// Depth-first, dirs first, but stop as soon as we hit caps.
|
|
79
|
+
for (const name of dirs) {
|
|
80
|
+
if (truncated || scanned >= maxFiles) return
|
|
81
|
+
if (depth <= 0) continue
|
|
82
|
+
const sub = joinRemotePath(dir, name)
|
|
83
|
+
if (isIgnored(name, true)) continue
|
|
84
|
+
await walk(sub, depth - 1)
|
|
85
|
+
}
|
|
86
|
+
if (truncated || scanned >= maxFiles) return
|
|
87
|
+
|
|
88
|
+
const fileNames = entries
|
|
89
|
+
.filter((e) => !(e.attrs && e.attrs.isDirectory && e.attrs.isDirectory()))
|
|
90
|
+
.map((e) => String(e.filename))
|
|
91
|
+
.filter((n) => n !== '.' && n !== '..')
|
|
92
|
+
await mapLimit(fileNames, 4, async (name) => {
|
|
93
|
+
if (truncated || scanned >= maxFiles || matches.length >= maxMatches) return
|
|
94
|
+
if (glob && !matchGlob(name, glob)) return
|
|
95
|
+
if (isIgnored(name, false)) return
|
|
96
|
+
const fp = joinRemotePath(dir, name)
|
|
97
|
+
scanned++
|
|
98
|
+
let st
|
|
99
|
+
try { st = await sftp.stat(fp) } catch { return }
|
|
100
|
+
if (st.size > maxScanBytes) return
|
|
101
|
+
let buf
|
|
102
|
+
try { buf = await sftp.readFile(fp) } catch { return }
|
|
103
|
+
if (looksBinary(buf)) return
|
|
104
|
+
const content = buf.toString('utf8').replace(/\r\n/g, '\n')
|
|
105
|
+
const lines = content.split('\n')
|
|
106
|
+
const hitLines = new Set()
|
|
107
|
+
for (let i = 0; i < lines.length; i++) {
|
|
108
|
+
if (regex.test(lines[i])) {
|
|
109
|
+
regex.lastIndex = 0
|
|
110
|
+
for (let c = Math.max(0, i - contextLines); c <= Math.min(lines.length - 1, i + contextLines); c++) {
|
|
111
|
+
hitLines.add(c)
|
|
112
|
+
}
|
|
113
|
+
if (matches.length >= maxMatches) { truncated = true; return }
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (hitLines.size) {
|
|
117
|
+
for (const ln of [...hitLines].sort((a, b) => a - b)) {
|
|
118
|
+
if (matches.length >= maxMatches) { truncated = true; return }
|
|
119
|
+
matches.push({ path: fp, line: ln + 1, text: lines[ln] })
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
})
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
await walk(rootDir, maxDepth)
|
|
126
|
+
return { matches, scanned, truncated }
|
|
127
|
+
}
|
package/lib/sshconfig.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// dsh-remote — ~/.ssh/config parser for the "import from ssh config" feature.
|
|
2
|
+
// Pure parsing (no auth material is ever read: only Host/HostName/User/Port/
|
|
3
|
+
// IdentityFile/ProxyJump). IdentityFile is imported as a PATH reference only —
|
|
4
|
+
// the plugin never auto-reads keys (v0.5.5 policy).
|
|
5
|
+
import { readFileSync } from 'node:fs'
|
|
6
|
+
import { homedir } from 'node:os'
|
|
7
|
+
import path from 'node:path'
|
|
8
|
+
|
|
9
|
+
export const sshConfigPath = () => path.join(homedir(), '.ssh', 'config')
|
|
10
|
+
|
|
11
|
+
/** Read the user's ~/.ssh/config text ('' when absent/unreadable). */
|
|
12
|
+
export function readSshConfigText(filePath = sshConfigPath()) {
|
|
13
|
+
try {
|
|
14
|
+
return readFileSync(filePath, 'utf8')
|
|
15
|
+
} catch {
|
|
16
|
+
return ''
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Parse OpenSSH config text into host entries.
|
|
22
|
+
* @returns {Array<{host: string, hostName: string, user: string, port: number, identityFile: string, proxyJump: string}>}
|
|
23
|
+
* host = alias pattern (may be '*' / '!foo' / comma list — caller filters).
|
|
24
|
+
*/
|
|
25
|
+
export function parseSshConfig(text) {
|
|
26
|
+
const entries = []
|
|
27
|
+
let cur = null
|
|
28
|
+
const lines = String(text || '').split('\n')
|
|
29
|
+
for (const raw of lines) {
|
|
30
|
+
const line = raw.trim()
|
|
31
|
+
if (!line || line.startsWith('#') || line.startsWith('Include ')) continue
|
|
32
|
+
const m = line.match(/^(\S+)\s+(.*)$/)
|
|
33
|
+
if (!m) continue
|
|
34
|
+
const key = m[1].toLowerCase()
|
|
35
|
+
const val = m[2].trim()
|
|
36
|
+
if (key === 'host') {
|
|
37
|
+
cur = { host: val, hostName: '', user: '', port: 22, identityFile: '', proxyJump: '' }
|
|
38
|
+
entries.push(cur)
|
|
39
|
+
continue
|
|
40
|
+
}
|
|
41
|
+
if (!cur) continue
|
|
42
|
+
if (key === 'hostname') cur.hostName = val
|
|
43
|
+
else if (key === 'user') cur.user = val
|
|
44
|
+
else if (key === 'port') cur.port = Number(val) || 22
|
|
45
|
+
else if (key === 'identityfile') cur.identityFile = val
|
|
46
|
+
else if (key === 'proxyjump') cur.proxyJump = val
|
|
47
|
+
}
|
|
48
|
+
return entries
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Entries a user can actually import: concrete aliases (skip wildcards). */
|
|
52
|
+
export function importableEntries(text) {
|
|
53
|
+
return parseSshConfig(text).filter((e) => {
|
|
54
|
+
if (!e.host || e.host.includes('*') || e.host.includes('!') || e.host.includes(',')) return false
|
|
55
|
+
return e.hostName || e.user || e.port !== 22 || e.identityFile || e.proxyJump
|
|
56
|
+
})
|
|
57
|
+
}
|
package/lib/sync.js
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// dsh-remote — three-way conflict-aware mirror sync (remote ⇄ local mirror).
|
|
2
|
+
//
|
|
3
|
+
// Why three-way: plain rsync-style pull/push silently clobbers whichever side
|
|
4
|
+
// changed while the other side was edited. We keep a per-mirror snapshot of the
|
|
5
|
+
// last-synced remote state (`.dsh-remote-sync-state.json`: relPath →
|
|
6
|
+
// {mtime,size}) and compare remote (R), local (L) and last-synced (S):
|
|
7
|
+
//
|
|
8
|
+
// R==S && L==S → unchanged, skip
|
|
9
|
+
// R!=S && L==S → remote changed → pull it (or push when pushing)
|
|
10
|
+
// R==S && L!=S → local changed, remote untouched
|
|
11
|
+
// • pull (rw_sync): would clobber local edit → conflict
|
|
12
|
+
// • push (rw_push): normal upload
|
|
13
|
+
// R!=S && L!=S && R==L → both changed to the same bytes → skip
|
|
14
|
+
// R!=S && L!=S && R!=L → BOTH changed differently → conflict (never clobber)
|
|
15
|
+
//
|
|
16
|
+
// `force: true` downgrades every conflict to a plain overwrite (rsync behavior).
|
|
17
|
+
// `dryRun: true` computes the full plan without writing anything.
|
|
18
|
+
// After a successful pull the local mtime is aligned to the remote mtime, and
|
|
19
|
+
// after a successful push the local mtime is aligned to the remote mtime the
|
|
20
|
+
// server assigned — so the invariant L==R==S holds and the next run is cheap.
|
|
21
|
+
import { readdirSync, statSync, mkdirSync, existsSync, readFileSync, writeFileSync, utimesSync, renameSync } from 'node:fs'
|
|
22
|
+
import path from 'node:path'
|
|
23
|
+
import { joinRemotePath, relPathUnder } from './paths.js'
|
|
24
|
+
|
|
25
|
+
function mapLimit(items, limit, fn) {
|
|
26
|
+
const limitN = Math.max(1, Math.min(Math.floor(limit) || 1, items.length))
|
|
27
|
+
let next = 0
|
|
28
|
+
const worker = async () => {
|
|
29
|
+
while (next < items.length) {
|
|
30
|
+
const i = next++
|
|
31
|
+
try { await fn(items[i], i) } catch { /* per-item errors are the fn's concern */ }
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
const workers = []
|
|
35
|
+
for (let w = 0; w < limitN; w++) workers.push(worker())
|
|
36
|
+
return Promise.all(workers)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const keyOf = (size, mtime) => `${size}:${Math.floor(mtime || 0)}`
|
|
40
|
+
const localKey = (st) => (st ? keyOf(st.size, st.mtimeMs / 1000) : null)
|
|
41
|
+
const remoteKey = (st) => (st ? keyOf(st.size, st.mtime) : null)
|
|
42
|
+
|
|
43
|
+
/** Recursively pull remote → local mirror (three-way, ignore-aware). */
|
|
44
|
+
export async function syncTree(sftp, remoteDir, localDir, opts = {}) {
|
|
45
|
+
const {
|
|
46
|
+
maxDepth = 5,
|
|
47
|
+
maxFiles = 500,
|
|
48
|
+
maxFileBytes = 0,
|
|
49
|
+
isIgnored = () => false,
|
|
50
|
+
dryRun = false,
|
|
51
|
+
force = false,
|
|
52
|
+
state = {},
|
|
53
|
+
} = opts
|
|
54
|
+
const stats = { files: 0, dirs: 0, skippedUnchanged: 0, skippedLarge: 0, conflicts: [], staleRemote: 0, touched: [] }
|
|
55
|
+
const nextState = { ...state }
|
|
56
|
+
|
|
57
|
+
const walk = async (rDir, lDir, depth) => {
|
|
58
|
+
const entries = await sftp.readdir(rDir).then((l) => l || [], () => [])
|
|
59
|
+
// Directories first (sequential, depth-bounded) so every file has a home.
|
|
60
|
+
for (const e of entries) {
|
|
61
|
+
const name = String(e.filename)
|
|
62
|
+
if (name === '.' || name === '..') continue
|
|
63
|
+
const isDir = !!(e.attrs && e.attrs.isDirectory && e.attrs.isDirectory())
|
|
64
|
+
if (!isDir) continue
|
|
65
|
+
if (depth <= 0 || stats.files >= maxFiles) continue
|
|
66
|
+
const rp = joinRemotePath(rDir, name)
|
|
67
|
+
const lp = path.join(lDir, name)
|
|
68
|
+
const rel = relPathUnder(remoteDir, rp) || name
|
|
69
|
+
if (isIgnored(rel, true)) continue
|
|
70
|
+
if (!existsSync(lp)) mkdirSync(lp, { recursive: true })
|
|
71
|
+
await walk(rp, lp, depth - 1)
|
|
72
|
+
stats.dirs++
|
|
73
|
+
if (stats.files >= maxFiles) break
|
|
74
|
+
}
|
|
75
|
+
if (stats.files >= maxFiles) return
|
|
76
|
+
|
|
77
|
+
const files = entries.filter(
|
|
78
|
+
(e) => !(e.attrs && e.attrs.isDirectory && e.attrs.isDirectory()) &&
|
|
79
|
+
String(e.filename) !== '.' && String(e.filename) !== '..',
|
|
80
|
+
)
|
|
81
|
+
await mapLimit(files, 4, async (e) => {
|
|
82
|
+
if (stats.files >= maxFiles) return
|
|
83
|
+
const name = String(e.filename)
|
|
84
|
+
const rp = joinRemotePath(rDir, name)
|
|
85
|
+
const lp = path.join(lDir, name)
|
|
86
|
+
const rel = relPathUnder(remoteDir, rp) || name
|
|
87
|
+
if (isIgnored(rel, false)) return
|
|
88
|
+
let r
|
|
89
|
+
try { r = await sftp.stat(rp) } catch { return }
|
|
90
|
+
if (maxFileBytes > 0 && r.size > maxFileBytes) { stats.skippedLarge++; return }
|
|
91
|
+
const rk = remoteKey(r)
|
|
92
|
+
const sk = state[rel] ? keyOf(state[rel].size, state[rel].mtime) : null
|
|
93
|
+
let lk = null
|
|
94
|
+
if (existsSync(lp)) {
|
|
95
|
+
try { lk = localKey(statSync(lp)) } catch { lk = null }
|
|
96
|
+
}
|
|
97
|
+
if (sk && rk === sk && lk && lk === sk) { stats.skippedUnchanged++; return }
|
|
98
|
+
if (!force && sk && rk !== sk && lk && lk !== sk && rk !== lk) {
|
|
99
|
+
stats.conflicts.push({ path: rp, reason: 'both-modified(远端与本地都改过,跳过不覆盖)' })
|
|
100
|
+
return
|
|
101
|
+
}
|
|
102
|
+
if (!force && sk && rk === sk && lk && lk !== sk) {
|
|
103
|
+
// Pull would clobber a local edit; the remote never changed.
|
|
104
|
+
stats.conflicts.push({ path: rp, reason: 'local-modified(本地改过而远端未变;如需覆盖请用 force / rw_push)' })
|
|
105
|
+
return
|
|
106
|
+
}
|
|
107
|
+
if (dryRun) {
|
|
108
|
+
stats.files++
|
|
109
|
+
stats.touched.push(rp)
|
|
110
|
+
nextState[rel] = { size: r.size, mtime: r.mtime }
|
|
111
|
+
return
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
const buf = await sftp.readFile(rp)
|
|
115
|
+
writeFileSync(lp, buf)
|
|
116
|
+
try { utimesSync(lp, new Date(r.mtime * 1000), new Date(r.mtime * 1000)) } catch {}
|
|
117
|
+
stats.files++
|
|
118
|
+
stats.touched.push(rp)
|
|
119
|
+
nextState[rel] = { size: r.size, mtime: r.mtime }
|
|
120
|
+
} catch { /* skip unreadable */ }
|
|
121
|
+
})
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
await walk(remoteDir, localDir, maxDepth)
|
|
125
|
+
// Informational: remote entries in the snapshot that no longer exist remotely.
|
|
126
|
+
for (const rel of Object.keys(state)) {
|
|
127
|
+
if (state[rel] && !nextState[rel]) stats.staleRemote++
|
|
128
|
+
}
|
|
129
|
+
return { stats, nextState }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Recursively push local mirror → remote (three-way, ignore-aware). */
|
|
133
|
+
export async function pushTree(sftp, localDir, remoteDir, opts = {}) {
|
|
134
|
+
const {
|
|
135
|
+
maxFiles = 500,
|
|
136
|
+
maxFileBytes = 0,
|
|
137
|
+
isIgnored = () => false,
|
|
138
|
+
dryRun = false,
|
|
139
|
+
force = false,
|
|
140
|
+
state = {},
|
|
141
|
+
} = opts
|
|
142
|
+
const stats = { files: 0, dirs: 0, skippedUnchanged: 0, skippedLarge: 0, conflicts: [], staleLocal: 0, pushed: [] }
|
|
143
|
+
const nextState = { ...state }
|
|
144
|
+
|
|
145
|
+
const walk = async (lDir, rDir) => {
|
|
146
|
+
const entries = readdirSync(lDir, { withFileTypes: true }).filter(
|
|
147
|
+
(e) => e.name !== '.dsh-remote-meta.json' && e.name !== '.dsh-remote-sync-state.json',
|
|
148
|
+
)
|
|
149
|
+
for (const e of entries) {
|
|
150
|
+
if (!e.isDirectory()) continue
|
|
151
|
+
if (stats.files >= maxFiles) break
|
|
152
|
+
const rp = joinRemotePath(rDir, e.name)
|
|
153
|
+
const rel = relPathUnder(remoteDir, rp) || e.name
|
|
154
|
+
if (isIgnored(rel, true)) continue
|
|
155
|
+
try { await sftp.mkdir(rp) } catch { /* already exists */ }
|
|
156
|
+
stats.dirs++
|
|
157
|
+
await walk(path.join(lDir, e.name), rp)
|
|
158
|
+
if (stats.files >= maxFiles) break
|
|
159
|
+
}
|
|
160
|
+
if (stats.files >= maxFiles) return
|
|
161
|
+
|
|
162
|
+
const files = entries.filter((e) => !e.isDirectory())
|
|
163
|
+
await mapLimit(files, 4, async (e) => {
|
|
164
|
+
if (stats.files >= maxFiles) return
|
|
165
|
+
const lp = path.join(lDir, e.name)
|
|
166
|
+
const rp = joinRemotePath(rDir, e.name)
|
|
167
|
+
const rel = relPathUnder(remoteDir, rp) || e.name
|
|
168
|
+
if (isIgnored(rel, false)) return
|
|
169
|
+
let l
|
|
170
|
+
try { l = statSync(lp) } catch { return }
|
|
171
|
+
if (maxFileBytes > 0 && l.size > maxFileBytes) { stats.skippedLarge++; return }
|
|
172
|
+
const lk = localKey(l)
|
|
173
|
+
let r = null
|
|
174
|
+
try { r = await sftp.stat(rp) } catch { /* absent → upload */ }
|
|
175
|
+
const rk = remoteKey(r)
|
|
176
|
+
const sk = state[rel] ? keyOf(state[rel].size, state[rel].mtime) : null
|
|
177
|
+
if (r && sk && lk === sk && rk === sk) { stats.skippedUnchanged++; return }
|
|
178
|
+
if (r && !force && sk && rk !== sk && lk !== sk && rk !== lk) {
|
|
179
|
+
stats.conflicts.push({ path: rp, reason: 'both-modified(远端与本地都改过,跳过不覆盖)' })
|
|
180
|
+
return
|
|
181
|
+
}
|
|
182
|
+
if (r && !force && sk && rk !== sk && lk === sk) {
|
|
183
|
+
stats.conflicts.push({ path: rp, reason: 'remote-modified(远端改过而本地未变;如需覆盖请用 force / 先 rw_sync)' })
|
|
184
|
+
return
|
|
185
|
+
}
|
|
186
|
+
if (r && !force && sk && rk === sk && lk && lk !== sk) {
|
|
187
|
+
// Normal case: local edited since sync, remote untouched → push.
|
|
188
|
+
}
|
|
189
|
+
if (dryRun) {
|
|
190
|
+
stats.files++
|
|
191
|
+
stats.pushed.push(rp)
|
|
192
|
+
return
|
|
193
|
+
}
|
|
194
|
+
try {
|
|
195
|
+
const buf = readFileSync(lp)
|
|
196
|
+
await sftp.writeFile(rp, buf)
|
|
197
|
+
// Align local mtime to whatever the remote server assigned, so the
|
|
198
|
+
// L==R==S invariant holds for the next run.
|
|
199
|
+
let r2 = null
|
|
200
|
+
try { r2 = await sftp.stat(rp) } catch {}
|
|
201
|
+
const rmt = r2 ? r2.mtime : Math.floor(Date.now() / 1000)
|
|
202
|
+
try { utimesSync(lp, new Date(rmt * 1000), new Date(rmt * 1000)) } catch {}
|
|
203
|
+
stats.files++
|
|
204
|
+
stats.pushed.push(rp)
|
|
205
|
+
nextState[rel] = { size: l.size, mtime: rmt }
|
|
206
|
+
} catch { /* skip unwritable */ }
|
|
207
|
+
})
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
await walk(localDir, remoteDir)
|
|
211
|
+
for (const rel of Object.keys(state)) {
|
|
212
|
+
if (state[rel] && !nextState[rel]) stats.staleLocal++
|
|
213
|
+
}
|
|
214
|
+
return { stats, nextState }
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Load the sync-state snapshot from a mirror dir (or {}). */
|
|
218
|
+
export function loadSyncState(localDir) {
|
|
219
|
+
try {
|
|
220
|
+
const j = JSON.parse(readFileSync(path.join(localDir, '.dsh-remote-sync-state.json'), 'utf8'))
|
|
221
|
+
if (j && typeof j === 'object' && !Array.isArray(j)) return j
|
|
222
|
+
} catch {}
|
|
223
|
+
return {}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Persist the snapshot (atomic-ish: tmp + rename). */
|
|
227
|
+
export function saveSyncState(localDir, state) {
|
|
228
|
+
try {
|
|
229
|
+
mkdirSync(localDir, { recursive: true })
|
|
230
|
+
const tmp = path.join(localDir, '.dsh-remote-sync-state.json.tmp')
|
|
231
|
+
writeFileSync(tmp, JSON.stringify(state, null, 1))
|
|
232
|
+
renameSync(tmp, path.join(localDir, '.dsh-remote-sync-state.json'))
|
|
233
|
+
} catch { /* best effort */ }
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Push ONE local mirror file back to the remote (used by auto-push).
|
|
237
|
+
* Same three-way guard as pushTree: never clobbers a remote change. Returns
|
|
238
|
+
* { status: 'pushed'|'unchanged'|'conflict'|'skipped'|'missing' } and the
|
|
239
|
+
* updated state delta ({rel: {size, mtime}} when pushed). */
|
|
240
|
+
export async function pushOneFile(sftp, localDir, remoteDir, rel, opts = {}) {
|
|
241
|
+
const { maxFileBytes = 0, isIgnored = () => false, force = false, state = {} } = opts
|
|
242
|
+
if (isIgnored(rel, false)) return { status: 'skipped' }
|
|
243
|
+
const lp = path.join(localDir, rel.replace(/\//g, path.sep))
|
|
244
|
+
const rp = joinRemotePath(remoteDir, rel)
|
|
245
|
+
let l
|
|
246
|
+
try { l = statSync(lp) } catch { return { status: 'missing' } }
|
|
247
|
+
if (maxFileBytes > 0 && l.size > maxFileBytes) return { status: 'skipped' }
|
|
248
|
+
const lk = localKey(l)
|
|
249
|
+
let r = null
|
|
250
|
+
try { r = await sftp.stat(rp) } catch { /* absent → upload */ }
|
|
251
|
+
const rk = remoteKey(r)
|
|
252
|
+
const sk = state[rel] ? keyOf(state[rel].size, state[rel].mtime) : null
|
|
253
|
+
if (r && sk && lk === sk && rk === sk) return { status: 'unchanged' }
|
|
254
|
+
if (r && !force && sk && rk !== sk && lk !== sk && rk !== lk) {
|
|
255
|
+
return { status: 'conflict', reason: 'both-modified' }
|
|
256
|
+
}
|
|
257
|
+
if (r && !force && sk && rk !== sk && lk === sk) {
|
|
258
|
+
return { status: 'conflict', reason: 'remote-modified' }
|
|
259
|
+
}
|
|
260
|
+
try {
|
|
261
|
+
await sftp.writeFile(rp, readFileSync(lp))
|
|
262
|
+
let r2 = null
|
|
263
|
+
try { r2 = await sftp.stat(rp) } catch {}
|
|
264
|
+
const rmt = r2 ? r2.mtime : Math.floor(Date.now() / 1000)
|
|
265
|
+
try { utimesSync(lp, new Date(rmt * 1000), new Date(rmt * 1000)) } catch {}
|
|
266
|
+
return { status: 'pushed', state: { [rel]: { size: l.size, mtime: rmt } } }
|
|
267
|
+
} catch (e) {
|
|
268
|
+
return { status: 'skipped', error: String((e && e.message) || e) }
|
|
269
|
+
}
|
|
270
|
+
}
|