@yuuki824/kanshi 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +34 -0
- package/Dockerfile +53 -0
- package/LICENSE +661 -0
- package/README.md +219 -0
- package/bin/kanshi.js +63 -0
- package/docker-compose.yml +84 -0
- package/go.mod +3 -0
- package/internal/access/access.go +275 -0
- package/internal/access/listeners.go +229 -0
- package/internal/config/config.go +256 -0
- package/internal/config/defaults_other.go +5 -0
- package/internal/config/defaults_windows.go +4 -0
- package/internal/config/envfile.go +147 -0
- package/internal/dockerstats/dial.go +32 -0
- package/internal/dockerstats/dial_other.go +13 -0
- package/internal/dockerstats/dial_windows.go +62 -0
- package/internal/dockerstats/dockerstats.go +626 -0
- package/internal/roots/roots.go +175 -0
- package/internal/roots/roots_linux.go +132 -0
- package/internal/roots/roots_windows.go +74 -0
- package/internal/server/access.go +155 -0
- package/internal/server/server.go +523 -0
- package/internal/server/static.go +165 -0
- package/internal/storage/cache.go +283 -0
- package/internal/storage/dir_linux.go +177 -0
- package/internal/storage/dir_windows.go +244 -0
- package/internal/storage/statat_fast.go +21 -0
- package/internal/storage/statat_other.go +11 -0
- package/internal/storage/storage.go +598 -0
- package/internal/storage/sysnum_amd64.go +7 -0
- package/internal/storage/sysnum_generic.go +7 -0
- package/internal/storage/walk.go +342 -0
- package/internal/vitals/vitals.go +246 -0
- package/internal/vitals/vitals_linux.go +548 -0
- package/internal/vitals/vitals_windows.go +286 -0
- package/main.go +129 -0
- package/main_other.go +11 -0
- package/main_windows.go +29 -0
- package/package.json +49 -0
- package/web/app.js +835 -0
- package/web/apple-touch-icon.png +0 -0
- package/web/icon.svg +6 -0
- package/web/index.html +151 -0
- package/web/style.css +356 -0
- package/web/treemap.js +217 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
package storage
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"bufio"
|
|
5
|
+
"encoding/gob"
|
|
6
|
+
"errors"
|
|
7
|
+
"io/fs"
|
|
8
|
+
"os"
|
|
9
|
+
"path/filepath"
|
|
10
|
+
"slices"
|
|
11
|
+
"strings"
|
|
12
|
+
"sync"
|
|
13
|
+
"time"
|
|
14
|
+
|
|
15
|
+
"github.com/rene-roid/kanshi/internal/roots"
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
// Bumped whenever the file changes shape. This is a cache, so an old one is
|
|
19
|
+
// dropped and rebuilt rather than migrated.
|
|
20
|
+
const cacheVersion = 1
|
|
21
|
+
|
|
22
|
+
// sized is what the cache knows about one folder.
|
|
23
|
+
type sized struct {
|
|
24
|
+
size int64
|
|
25
|
+
unreadable int
|
|
26
|
+
at time.Time
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// cache holds every folder a walk has sized, grouped by parent so a listing's
|
|
30
|
+
// subfolders are one map lookup. Paths are roots.Key form: cleaned, and
|
|
31
|
+
// lower-cased on Windows.
|
|
32
|
+
//
|
|
33
|
+
// It is small enough to keep in memory whole (one entry per folder, a few
|
|
34
|
+
// thousand for all of / on the homeserver), and is written back to one file
|
|
35
|
+
// after each walk rather than entry by entry.
|
|
36
|
+
type cache struct {
|
|
37
|
+
path string // "" keeps it in memory only
|
|
38
|
+
|
|
39
|
+
mu sync.Mutex
|
|
40
|
+
dirs map[string]map[string]sized // parent key → name key → size
|
|
41
|
+
dirty bool
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// cacheFile is the on-disk form. gob needs exported fields.
|
|
45
|
+
type cacheFile struct {
|
|
46
|
+
Version int
|
|
47
|
+
Dirs map[string]map[string]cacheEntry
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
type cacheEntry struct {
|
|
51
|
+
Size int64
|
|
52
|
+
Unreadable int
|
|
53
|
+
SizedAt int64 // unix seconds
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// openCache loads the cache at path, or starts an in-memory one when path is
|
|
57
|
+
// empty. A file that is missing, from another version or not a cache at all
|
|
58
|
+
// starts it empty. It is then written straight away, so a path that cannot be
|
|
59
|
+
// written to is reported now rather than after the first walk.
|
|
60
|
+
func openCache(path string) (*cache, error) {
|
|
61
|
+
c := &cache{path: path, dirs: map[string]map[string]sized{}}
|
|
62
|
+
if path == "" {
|
|
63
|
+
return c, nil
|
|
64
|
+
}
|
|
65
|
+
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
66
|
+
return nil, err
|
|
67
|
+
}
|
|
68
|
+
f, err := os.Open(path)
|
|
69
|
+
if err == nil {
|
|
70
|
+
var file cacheFile
|
|
71
|
+
ok := gob.NewDecoder(bufio.NewReader(f)).Decode(&file) == nil && file.Version == cacheVersion
|
|
72
|
+
f.Close()
|
|
73
|
+
if ok {
|
|
74
|
+
for parent, children := range file.Dirs {
|
|
75
|
+
m := make(map[string]sized, len(children))
|
|
76
|
+
for name, e := range children {
|
|
77
|
+
m[name] = sized{size: e.Size, unreadable: e.Unreadable, at: time.Unix(e.SizedAt, 0)}
|
|
78
|
+
}
|
|
79
|
+
c.dirs[parent] = m
|
|
80
|
+
}
|
|
81
|
+
return c, nil
|
|
82
|
+
}
|
|
83
|
+
} else if !errors.Is(err, fs.ErrNotExist) {
|
|
84
|
+
return nil, err
|
|
85
|
+
}
|
|
86
|
+
c.dirty = true
|
|
87
|
+
if err := c.save(); err != nil {
|
|
88
|
+
return nil, err
|
|
89
|
+
}
|
|
90
|
+
return c, nil
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// close writes out anything a listing changed since the last walk.
|
|
94
|
+
func (c *cache) close() error { return c.save() }
|
|
95
|
+
|
|
96
|
+
// save writes the cache if it has changed. It goes to a temporary file that
|
|
97
|
+
// then replaces the old one, so a crash mid-write leaves the previous cache
|
|
98
|
+
// rather than half of this one.
|
|
99
|
+
func (c *cache) save() error {
|
|
100
|
+
c.mu.Lock()
|
|
101
|
+
if c.path == "" || !c.dirty {
|
|
102
|
+
c.mu.Unlock()
|
|
103
|
+
return nil
|
|
104
|
+
}
|
|
105
|
+
file := cacheFile{Version: cacheVersion, Dirs: make(map[string]map[string]cacheEntry, len(c.dirs))}
|
|
106
|
+
for parent, children := range c.dirs {
|
|
107
|
+
m := make(map[string]cacheEntry, len(children))
|
|
108
|
+
for name, s := range children {
|
|
109
|
+
m[name] = cacheEntry{Size: s.size, Unreadable: s.unreadable, SizedAt: s.at.Unix()}
|
|
110
|
+
}
|
|
111
|
+
file.Dirs[parent] = m
|
|
112
|
+
}
|
|
113
|
+
c.dirty = false
|
|
114
|
+
c.mu.Unlock()
|
|
115
|
+
|
|
116
|
+
tmp, err := os.CreateTemp(filepath.Dir(c.path), filepath.Base(c.path)+".*")
|
|
117
|
+
if err != nil {
|
|
118
|
+
c.markDirty()
|
|
119
|
+
return err
|
|
120
|
+
}
|
|
121
|
+
w := bufio.NewWriter(tmp)
|
|
122
|
+
err = gob.NewEncoder(w).Encode(&file)
|
|
123
|
+
if err == nil {
|
|
124
|
+
err = w.Flush()
|
|
125
|
+
}
|
|
126
|
+
if cerr := tmp.Close(); err == nil {
|
|
127
|
+
err = cerr
|
|
128
|
+
}
|
|
129
|
+
if err == nil {
|
|
130
|
+
err = os.Rename(tmp.Name(), c.path)
|
|
131
|
+
}
|
|
132
|
+
if err != nil {
|
|
133
|
+
os.Remove(tmp.Name())
|
|
134
|
+
c.markDirty()
|
|
135
|
+
}
|
|
136
|
+
return err
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
func (c *cache) markDirty() {
|
|
140
|
+
c.mu.Lock()
|
|
141
|
+
c.dirty = true
|
|
142
|
+
c.mu.Unlock()
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// get returns what is cached for one folder.
|
|
146
|
+
func (c *cache) get(key string) (sized, bool) {
|
|
147
|
+
parent, name := splitKey(key)
|
|
148
|
+
c.mu.Lock()
|
|
149
|
+
defer c.mu.Unlock()
|
|
150
|
+
s, ok := c.dirs[parent][name]
|
|
151
|
+
return s, ok
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// children returns every cached subfolder of dir, by name key. The map is a
|
|
155
|
+
// copy, so the caller can hold on to it while walks carry on.
|
|
156
|
+
func (c *cache) children(dir string) map[string]sized {
|
|
157
|
+
c.mu.Lock()
|
|
158
|
+
defer c.mu.Unlock()
|
|
159
|
+
out := make(map[string]sized, len(c.dirs[dir]))
|
|
160
|
+
for name, s := range c.dirs[dir] {
|
|
161
|
+
out[name] = s
|
|
162
|
+
}
|
|
163
|
+
return out
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// put records one folder's size. It is saved with the next walk or on close:
|
|
167
|
+
// a listing can always work it out again.
|
|
168
|
+
func (c *cache) put(key string, s sized) {
|
|
169
|
+
c.mu.Lock()
|
|
170
|
+
defer c.mu.Unlock()
|
|
171
|
+
c.set(key, s)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// set records one folder's size. The caller holds mu.
|
|
175
|
+
func (c *cache) set(key string, s sized) {
|
|
176
|
+
parent, name := splitKey(key)
|
|
177
|
+
m := c.dirs[parent]
|
|
178
|
+
if m == nil {
|
|
179
|
+
m = map[string]sized{}
|
|
180
|
+
c.dirs[parent] = m
|
|
181
|
+
}
|
|
182
|
+
m[name] = s
|
|
183
|
+
c.dirty = true
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// store replaces everything cached at and below key with one walk's result,
|
|
187
|
+
// and saves the cache. Entries below the walk's depth are dropped rather than
|
|
188
|
+
// kept: they predate it, and a folder with no entry is simply walked again
|
|
189
|
+
// when it is opened. Other roots inside key are left alone, since the walk
|
|
190
|
+
// may not have entered them: /mnt/data is its own root, on its own
|
|
191
|
+
// filesystem, under /.
|
|
192
|
+
func (c *cache) store(key string, t *tree, at time.Time, rootKeys []string) error {
|
|
193
|
+
c.mu.Lock()
|
|
194
|
+
c.deleteSubtree(key, rootKeys)
|
|
195
|
+
// Breadth-first order puts every parent before its children, so each
|
|
196
|
+
// node's key is known by the time its children need it.
|
|
197
|
+
keys := make([]string, len(t.nodes))
|
|
198
|
+
keys[0] = key
|
|
199
|
+
for i, n := range t.nodes {
|
|
200
|
+
for _, ch := range n.children {
|
|
201
|
+
keys[ch] = joinKey(keys[i], nameKey(t.nodes[ch].name))
|
|
202
|
+
}
|
|
203
|
+
c.set(keys[i], sized{size: n.size, unreadable: int(n.unreadable), at: at})
|
|
204
|
+
}
|
|
205
|
+
c.mu.Unlock()
|
|
206
|
+
return c.save()
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// prune forgets the subfolders of dir that are no longer there, so a folder
|
|
210
|
+
// recreated under an old name is sized afresh. Other roots are kept: a drive
|
|
211
|
+
// mounted in dir is not one of its subfolders, but is still there.
|
|
212
|
+
func (c *cache) prune(dir string, cached map[string]sized, present map[string]bool, rootKeys []string) {
|
|
213
|
+
c.mu.Lock()
|
|
214
|
+
defer c.mu.Unlock()
|
|
215
|
+
for name := range cached {
|
|
216
|
+
if key := joinKey(dir, name); !present[name] && !slices.Contains(rootKeys, key) {
|
|
217
|
+
c.deleteSubtree(key, rootKeys)
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// deleteSubtree removes key and everything below it, except for the
|
|
223
|
+
// subtrees of any other root in rootKeys. The caller holds mu.
|
|
224
|
+
func (c *cache) deleteSubtree(key string, rootKeys []string) {
|
|
225
|
+
var keep []string
|
|
226
|
+
for _, r := range rootKeys {
|
|
227
|
+
if r != key && below(key, r) {
|
|
228
|
+
keep = append(keep, r)
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
parent, name := splitKey(key)
|
|
232
|
+
if m, ok := c.dirs[parent]; ok {
|
|
233
|
+
delete(m, name)
|
|
234
|
+
if len(m) == 0 {
|
|
235
|
+
delete(c.dirs, parent)
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
for p, m := range c.dirs {
|
|
239
|
+
if !below(key, p) || slices.ContainsFunc(keep, func(r string) bool { return below(r, p) }) {
|
|
240
|
+
continue
|
|
241
|
+
}
|
|
242
|
+
// A kept root's own entry sits under a parent inside key, which
|
|
243
|
+
// otherwise loses everything.
|
|
244
|
+
for name := range m {
|
|
245
|
+
if !slices.Contains(keep, joinKey(p, name)) {
|
|
246
|
+
delete(m, name)
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if len(m) == 0 {
|
|
250
|
+
delete(c.dirs, p)
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
c.dirty = true
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// below reports whether key is dir itself or somewhere inside it. Both are
|
|
257
|
+
// already in key form.
|
|
258
|
+
func below(dir, key string) bool {
|
|
259
|
+
if key == dir {
|
|
260
|
+
return true
|
|
261
|
+
}
|
|
262
|
+
if !strings.HasSuffix(dir, roots.Sep) {
|
|
263
|
+
dir += roots.Sep
|
|
264
|
+
}
|
|
265
|
+
return strings.HasPrefix(key, dir)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// splitKey divides a path key into its parent's key and its own name. A
|
|
269
|
+
// volume root such as "/" or "c:\" has no parent and is its own name.
|
|
270
|
+
func splitKey(key string) (parent, name string) {
|
|
271
|
+
parent = filepath.Dir(key)
|
|
272
|
+
if parent == key {
|
|
273
|
+
return "", key
|
|
274
|
+
}
|
|
275
|
+
return parent, filepath.Base(key)
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
func joinKey(dir, name string) string {
|
|
279
|
+
if strings.HasSuffix(dir, roots.Sep) {
|
|
280
|
+
return dir + name
|
|
281
|
+
}
|
|
282
|
+
return dir + roots.Sep + name
|
|
283
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
package storage
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"errors"
|
|
5
|
+
"syscall"
|
|
6
|
+
"time"
|
|
7
|
+
"unsafe"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
const pathSep = '/'
|
|
11
|
+
|
|
12
|
+
// normKey matches roots.Key for the clean paths the walk builds.
|
|
13
|
+
func normKey(p []byte) string { return string(p) }
|
|
14
|
+
|
|
15
|
+
func sameName(a, b string) bool { return a == b }
|
|
16
|
+
|
|
17
|
+
// nameKey is how one name appears in a cache key.
|
|
18
|
+
func nameKey(name string) string { return name }
|
|
19
|
+
|
|
20
|
+
// systemRoot is where Windows keeps its hard links. Linux has no such place:
|
|
21
|
+
// every multiply-linked file is spotted by its link count instead.
|
|
22
|
+
var systemRoot = func() string { return "" }
|
|
23
|
+
|
|
24
|
+
// rootDevice checks the root is a directory and returns the filesystem it is
|
|
25
|
+
// on, which the walk will not leave.
|
|
26
|
+
func rootDevice(path string) (uint64, error) {
|
|
27
|
+
var st syscall.Stat_t
|
|
28
|
+
if err := syscall.Stat(path, &st); err != nil {
|
|
29
|
+
return 0, err
|
|
30
|
+
}
|
|
31
|
+
if st.Mode&syscall.S_IFMT != syscall.S_IFDIR {
|
|
32
|
+
return 0, errors.New("not a directory")
|
|
33
|
+
}
|
|
34
|
+
return uint64(st.Dev), nil
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Linux dirent64: d_ino (8), d_off (8), d_reclen (2), d_type (1), d_name.
|
|
38
|
+
const (
|
|
39
|
+
direntReclen = 16
|
|
40
|
+
direntType = 18
|
|
41
|
+
direntName = 19
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
const (
|
|
45
|
+
atSymlinkNofollow = 0x100
|
|
46
|
+
atNoAutomount = 0x800
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
// A variable, not a constant: a negative constant cannot convert to uintptr.
|
|
50
|
+
var atFDCWD = -100
|
|
51
|
+
|
|
52
|
+
// dirReader lists directories with getdents64 and fstatat.
|
|
53
|
+
//
|
|
54
|
+
// The loop does no allocation per entry: records are parsed in place from one
|
|
55
|
+
// reused buffer, and each is stat'ed relative to its directory's descriptor
|
|
56
|
+
// with the NUL-terminated name straight out of that buffer, so the kernel
|
|
57
|
+
// resolves one path component rather than the whole path from the root.
|
|
58
|
+
type dirReader struct {
|
|
59
|
+
buf []byte
|
|
60
|
+
names []byte
|
|
61
|
+
entries []dirEntry
|
|
62
|
+
st syscall.Stat_t
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
func newDirReader() *dirReader { return &dirReader{buf: make([]byte, 32*1024)} }
|
|
66
|
+
|
|
67
|
+
// read lists one directory. path must be followed by a NUL in its backing
|
|
68
|
+
// array. The result is only valid until the next call.
|
|
69
|
+
func (r *dirReader) read(path []byte, _ bool) ([]dirEntry, int, error) {
|
|
70
|
+
fd, err := openDir(path)
|
|
71
|
+
if err != nil {
|
|
72
|
+
return nil, 0, err
|
|
73
|
+
}
|
|
74
|
+
defer syscall.Close(fd)
|
|
75
|
+
|
|
76
|
+
r.entries, r.names = r.entries[:0], r.names[:0]
|
|
77
|
+
seen := 0
|
|
78
|
+
for {
|
|
79
|
+
n, err := syscall.Getdents(fd, r.buf)
|
|
80
|
+
if err != nil || n <= 0 {
|
|
81
|
+
break // end of directory, or a read error mid-directory
|
|
82
|
+
}
|
|
83
|
+
for off := 0; off < n; {
|
|
84
|
+
reclen := int(*(*uint16)(unsafe.Pointer(&r.buf[off+direntReclen])))
|
|
85
|
+
typ := r.buf[off+direntType]
|
|
86
|
+
name := r.buf[off+direntName : off+reclen]
|
|
87
|
+
off += reclen
|
|
88
|
+
if end := indexNUL(name); end >= 0 {
|
|
89
|
+
name = name[:end]
|
|
90
|
+
}
|
|
91
|
+
seen++
|
|
92
|
+
// d_type comes back with the directory entry, so symlinks,
|
|
93
|
+
// sockets and devices are rejected without a stat at all.
|
|
94
|
+
if typ != syscall.DT_DIR && typ != syscall.DT_REG && typ != syscall.DT_UNKNOWN {
|
|
95
|
+
continue
|
|
96
|
+
}
|
|
97
|
+
if len(name) == 0 || (name[0] == '.' && (len(name) == 1 || (len(name) == 2 && name[1] == '.'))) {
|
|
98
|
+
continue
|
|
99
|
+
}
|
|
100
|
+
if statAt(fd, path, name, &r.st) != nil {
|
|
101
|
+
continue
|
|
102
|
+
}
|
|
103
|
+
var e dirEntry
|
|
104
|
+
switch r.st.Mode & syscall.S_IFMT {
|
|
105
|
+
case syscall.S_IFDIR:
|
|
106
|
+
e.dir, e.dev = true, uint64(r.st.Dev)
|
|
107
|
+
case syscall.S_IFREG:
|
|
108
|
+
e.size = int64(r.st.Blocks) * 512
|
|
109
|
+
if r.st.Nlink > 1 {
|
|
110
|
+
e.key = uint64(r.st.Dev)<<48 | uint64(r.st.Ino)
|
|
111
|
+
}
|
|
112
|
+
default:
|
|
113
|
+
continue
|
|
114
|
+
}
|
|
115
|
+
// Copied out because the next getdents call reuses buf.
|
|
116
|
+
start := len(r.names)
|
|
117
|
+
r.names = append(r.names, name...)
|
|
118
|
+
e.name = r.names[start:len(r.names):len(r.names)]
|
|
119
|
+
r.entries = append(r.entries, e)
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return r.entries, seen, nil
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// openDir opens a directory by a path that is followed by a NUL in its
|
|
126
|
+
// backing array, so the kernel reads it in place rather than from a copy.
|
|
127
|
+
func openDir(path []byte) (int, error) {
|
|
128
|
+
fd, _, errno := syscall.Syscall6(syscall.SYS_OPENAT, uintptr(atFDCWD), uintptr(unsafe.Pointer(&path[0])),
|
|
129
|
+
syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0, 0, 0)
|
|
130
|
+
if errno != 0 {
|
|
131
|
+
return -1, errno
|
|
132
|
+
}
|
|
133
|
+
return int(fd), nil
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
func indexNUL(b []byte) int {
|
|
137
|
+
for i, c := range b {
|
|
138
|
+
if c == 0 {
|
|
139
|
+
return i
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return -1
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// join builds a full path without path/filepath's Clean pass: every path here
|
|
146
|
+
// is an already-clean parent plus one getdents name.
|
|
147
|
+
func join(dir, name string) string {
|
|
148
|
+
if dir == "/" {
|
|
149
|
+
return "/" + name
|
|
150
|
+
}
|
|
151
|
+
return dir + "/" + name
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const (
|
|
155
|
+
ioprioWhoProcess = 1
|
|
156
|
+
ioprioClassBE = 2
|
|
157
|
+
ioprioClassShift = 13
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
// deprioritise puts the calling thread at the back of the queue for both the
|
|
161
|
+
// CPU (nice 19) and the disk (lowest best-effort I/O priority), so a rescan
|
|
162
|
+
// never makes the at-a-glance numbers stutter. The idle I/O class would be
|
|
163
|
+
// gentler still, but on a busy box it can starve the walk outright.
|
|
164
|
+
func deprioritise() {
|
|
165
|
+
_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
|
|
166
|
+
syscall.RawSyscall(syscall.SYS_IOPRIO_SET, ioprioWhoProcess, 0, ioprioClassBE<<ioprioClassShift|7)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// threadCPU is the user+system time of the calling thread, which the walk has
|
|
170
|
+
// to itself because it is locked to it.
|
|
171
|
+
func threadCPU() time.Duration {
|
|
172
|
+
var ru syscall.Rusage
|
|
173
|
+
if syscall.Getrusage(syscall.RUSAGE_THREAD, &ru) != nil {
|
|
174
|
+
return 0
|
|
175
|
+
}
|
|
176
|
+
return time.Duration(ru.Utime.Nano() + ru.Stime.Nano())
|
|
177
|
+
}
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
package storage
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"errors"
|
|
5
|
+
"os"
|
|
6
|
+
"strings"
|
|
7
|
+
"sync"
|
|
8
|
+
"syscall"
|
|
9
|
+
"time"
|
|
10
|
+
"unicode/utf16"
|
|
11
|
+
"unicode/utf8"
|
|
12
|
+
"unsafe"
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
const pathSep = '\\'
|
|
16
|
+
|
|
17
|
+
// normKey matches roots.Key: NTFS names compare case-insensitively.
|
|
18
|
+
func normKey(p []byte) string { return strings.ToLower(string(p)) }
|
|
19
|
+
|
|
20
|
+
// sameName lets a pasted c:\users\... find C:\Users\... on disk.
|
|
21
|
+
func sameName(a, b string) bool { return strings.EqualFold(a, b) }
|
|
22
|
+
|
|
23
|
+
// nameKey is how one name appears in a cache key, lower-cased like roots.Key.
|
|
24
|
+
func nameKey(name string) string { return strings.ToLower(name) }
|
|
25
|
+
|
|
26
|
+
// systemRoot is C:\Windows. Its component store hard-links most of System32,
|
|
27
|
+
// so files there are de-duplicated by file ID; anywhere else hard links are
|
|
28
|
+
// rare enough that tracking every file ID would cost more memory than it saves.
|
|
29
|
+
// A variable so tests can point it at a scratch directory.
|
|
30
|
+
var systemRoot = func() string { return os.Getenv("SystemRoot") }
|
|
31
|
+
|
|
32
|
+
func rootDevice(path string) (uint64, error) {
|
|
33
|
+
info, err := os.Stat(path)
|
|
34
|
+
if err != nil {
|
|
35
|
+
return 0, err
|
|
36
|
+
}
|
|
37
|
+
if !info.IsDir() {
|
|
38
|
+
return 0, errors.New("not a directory")
|
|
39
|
+
}
|
|
40
|
+
// Windows walks never cross into another volume anyway: mounted folders
|
|
41
|
+
// are reparse points, and those are skipped.
|
|
42
|
+
return 0, nil
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
var (
|
|
46
|
+
kernel32 = syscall.NewLazyDLL("kernel32.dll")
|
|
47
|
+
procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx")
|
|
48
|
+
procSetThreadPriority = kernel32.NewProc("SetThreadPriority")
|
|
49
|
+
procGetThreadTimes = kernel32.NewProc("GetThreadTimes")
|
|
50
|
+
|
|
51
|
+
advapi32 = syscall.NewLazyDLL("advapi32.dll")
|
|
52
|
+
procLookupPrivilegeValueW = advapi32.NewProc("LookupPrivilegeValueW")
|
|
53
|
+
procAdjustTokenPrivileges = advapi32.NewProc("AdjustTokenPrivileges")
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
const (
|
|
57
|
+
fileListDirectory = 0x0001
|
|
58
|
+
fileShareAll = syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | 0x4 // | FILE_SHARE_DELETE
|
|
59
|
+
fileFlagBackupSemantics = 0x02000000
|
|
60
|
+
fileIDBothDirectoryInfo = 10
|
|
61
|
+
errorNoMoreFiles = 18
|
|
62
|
+
|
|
63
|
+
attrDirectory = 0x10
|
|
64
|
+
attrReparsePoint = 0x400
|
|
65
|
+
// Junctions, symlinks and mounted folders all carry the name-surrogate
|
|
66
|
+
// bit: they point somewhere else, and following them would loop or count
|
|
67
|
+
// a directory twice. OneDrive placeholders do not, so they are walked.
|
|
68
|
+
reparseNameSurrogate = 0x20000000
|
|
69
|
+
|
|
70
|
+
// FILE_ID_BOTH_DIR_INFO field offsets.
|
|
71
|
+
offNext = 0
|
|
72
|
+
offAllocation = 48
|
|
73
|
+
offAttributes = 56
|
|
74
|
+
offNameLength = 60
|
|
75
|
+
offEaSize = 64 // holds the reparse tag when attrReparsePoint is set
|
|
76
|
+
offFileID = 96
|
|
77
|
+
offName = 104
|
|
78
|
+
|
|
79
|
+
currentThread = ^uintptr(1) // GetCurrentThread's pseudo-handle, -2
|
|
80
|
+
threadModeBackgroundBegin = 0x00010000
|
|
81
|
+
sePrivilegeEnabled = 0x2
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
// dirReader lists directories with GetFileInformationByHandleEx, which hands
|
|
85
|
+
// back names, attributes and allocation sizes for a whole batch of entries in
|
|
86
|
+
// one call — the same no-stat-per-file property the Linux reader gets from
|
|
87
|
+
// fstatat, and the size on disk rather than the logical length.
|
|
88
|
+
type dirReader struct {
|
|
89
|
+
buf []byte // 8-byte aligned, as the records inside it require
|
|
90
|
+
path16 []uint16
|
|
91
|
+
names []byte
|
|
92
|
+
entries []dirEntry
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
func newDirReader() *dirReader {
|
|
96
|
+
words := make([]uint64, 64*1024/8)
|
|
97
|
+
return &dirReader{buf: unsafe.Slice((*byte)(unsafe.Pointer(&words[0])), len(words)*8)}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
func (r *dirReader) read(path []byte, dedupe bool) ([]dirEntry, int, error) {
|
|
101
|
+
h, err := syscall.CreateFile(r.longPath(path), fileListDirectory, fileShareAll, nil,
|
|
102
|
+
syscall.OPEN_EXISTING, fileFlagBackupSemantics, 0)
|
|
103
|
+
if err != nil {
|
|
104
|
+
return nil, 0, err
|
|
105
|
+
}
|
|
106
|
+
defer syscall.CloseHandle(h)
|
|
107
|
+
|
|
108
|
+
r.entries, r.names = r.entries[:0], r.names[:0]
|
|
109
|
+
seen := 0
|
|
110
|
+
for first := true; ; first = false {
|
|
111
|
+
ok, _, e := procGetFileInformationByHandleEx.Call(uintptr(h), fileIDBothDirectoryInfo,
|
|
112
|
+
uintptr(unsafe.Pointer(&r.buf[0])), uintptr(len(r.buf)))
|
|
113
|
+
if ok == 0 {
|
|
114
|
+
if first && e != syscall.Errno(errorNoMoreFiles) {
|
|
115
|
+
return nil, 0, e // opened, but listing it was refused
|
|
116
|
+
}
|
|
117
|
+
break
|
|
118
|
+
}
|
|
119
|
+
for off := 0; ; {
|
|
120
|
+
rec := r.buf[off:]
|
|
121
|
+
next := u32(rec, offNext)
|
|
122
|
+
nameLen := int(u32(rec, offNameLength)) / 2
|
|
123
|
+
name := unsafe.Slice((*uint16)(unsafe.Pointer(&rec[offName])), nameLen)
|
|
124
|
+
r.add(name, rec, dedupe, &seen)
|
|
125
|
+
if next == 0 {
|
|
126
|
+
break
|
|
127
|
+
}
|
|
128
|
+
off += int(next)
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return r.entries, seen, nil
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
func (r *dirReader) add(name []uint16, rec []byte, dedupe bool, seen *int) {
|
|
135
|
+
if len(name) == 1 && name[0] == '.' || len(name) == 2 && name[0] == '.' && name[1] == '.' {
|
|
136
|
+
return
|
|
137
|
+
}
|
|
138
|
+
*seen++
|
|
139
|
+
attrs := u32(rec, offAttributes)
|
|
140
|
+
if attrs&attrReparsePoint != 0 && u32(rec, offEaSize)&reparseNameSurrogate != 0 {
|
|
141
|
+
return
|
|
142
|
+
}
|
|
143
|
+
var e dirEntry
|
|
144
|
+
if attrs&attrDirectory != 0 {
|
|
145
|
+
e.dir = true
|
|
146
|
+
} else {
|
|
147
|
+
e.size = *(*int64)(unsafe.Pointer(&rec[offAllocation]))
|
|
148
|
+
if dedupe {
|
|
149
|
+
e.key = *(*uint64)(unsafe.Pointer(&rec[offFileID]))
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
start := len(r.names)
|
|
153
|
+
r.names = appendUTF8(r.names, name)
|
|
154
|
+
e.name = r.names[start:len(r.names):len(r.names)]
|
|
155
|
+
r.entries = append(r.entries, e)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// longPath converts to UTF-16 with the \\?\ prefix, which lifts the 260
|
|
159
|
+
// character limit that a deep node_modules tree would otherwise hit.
|
|
160
|
+
func (r *dirReader) longPath(path []byte) *uint16 {
|
|
161
|
+
p := r.path16[:0]
|
|
162
|
+
s := string(path)
|
|
163
|
+
if strings.HasPrefix(s, `\\`) {
|
|
164
|
+
p = append(p, []uint16(utf16.Encode([]rune(`\\?\UNC\`)))...)
|
|
165
|
+
s = s[2:]
|
|
166
|
+
} else {
|
|
167
|
+
p = append(p, '\\', '\\', '?', '\\')
|
|
168
|
+
}
|
|
169
|
+
for _, c := range s {
|
|
170
|
+
p = utf16.AppendRune(p, c)
|
|
171
|
+
}
|
|
172
|
+
p = append(p, 0)
|
|
173
|
+
r.path16 = p
|
|
174
|
+
return &p[0]
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
func appendUTF8(dst []byte, name []uint16) []byte {
|
|
178
|
+
for i := 0; i < len(name); i++ {
|
|
179
|
+
c := rune(name[i])
|
|
180
|
+
if c < utf8.RuneSelf {
|
|
181
|
+
dst = append(dst, byte(c))
|
|
182
|
+
continue
|
|
183
|
+
}
|
|
184
|
+
if utf16.IsSurrogate(c) && i+1 < len(name) {
|
|
185
|
+
if pair := utf16.DecodeRune(c, rune(name[i+1])); pair != utf8.RuneError {
|
|
186
|
+
c = pair
|
|
187
|
+
i++
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
dst = utf8.AppendRune(dst, c)
|
|
191
|
+
}
|
|
192
|
+
return dst
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
func u32(b []byte, off int) uint32 { return *(*uint32)(unsafe.Pointer(&b[off])) }
|
|
196
|
+
|
|
197
|
+
var backupOnce sync.Once
|
|
198
|
+
|
|
199
|
+
// deprioritise puts the calling thread in background mode, which lowers its
|
|
200
|
+
// CPU, I/O and memory priority together — Windows' equivalent of nice 19 plus
|
|
201
|
+
// the lowest I/O class.
|
|
202
|
+
//
|
|
203
|
+
// It also enables SeBackupPrivilege where the token holds it (an elevated
|
|
204
|
+
// administrator), so folders whose ACLs shut out even administrators are
|
|
205
|
+
// still counted. That privilege only grants reading; nothing is ever opened
|
|
206
|
+
// for writing.
|
|
207
|
+
func deprioritise() {
|
|
208
|
+
procSetThreadPriority.Call(currentThread, threadModeBackgroundBegin)
|
|
209
|
+
backupOnce.Do(enableBackupPrivilege)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
func enableBackupPrivilege() {
|
|
213
|
+
var token syscall.Token
|
|
214
|
+
proc, _ := syscall.GetCurrentProcess()
|
|
215
|
+
if syscall.OpenProcessToken(proc, syscall.TOKEN_ADJUST_PRIVILEGES|syscall.TOKEN_QUERY, &token) != nil {
|
|
216
|
+
return
|
|
217
|
+
}
|
|
218
|
+
defer token.Close()
|
|
219
|
+
var privileges struct {
|
|
220
|
+
count uint32
|
|
221
|
+
luid struct {
|
|
222
|
+
low uint32
|
|
223
|
+
high int32
|
|
224
|
+
}
|
|
225
|
+
attributes uint32
|
|
226
|
+
}
|
|
227
|
+
name, _ := syscall.UTF16PtrFromString("SeBackupPrivilege")
|
|
228
|
+
if ok, _, _ := procLookupPrivilegeValueW.Call(0, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(&privileges.luid))); ok == 0 {
|
|
229
|
+
return
|
|
230
|
+
}
|
|
231
|
+
privileges.count, privileges.attributes = 1, sePrivilegeEnabled
|
|
232
|
+
procAdjustTokenPrivileges.Call(uintptr(token), 0, uintptr(unsafe.Pointer(&privileges)), 0, 0, 0)
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// threadCPU is the kernel+user time of the calling thread.
|
|
236
|
+
func threadCPU() time.Duration {
|
|
237
|
+
var created, exited, kernel, user syscall.Filetime
|
|
238
|
+
if ok, _, _ := procGetThreadTimes.Call(currentThread, uintptr(unsafe.Pointer(&created)),
|
|
239
|
+
uintptr(unsafe.Pointer(&exited)), uintptr(unsafe.Pointer(&kernel)), uintptr(unsafe.Pointer(&user))); ok == 0 {
|
|
240
|
+
return 0
|
|
241
|
+
}
|
|
242
|
+
ticks := func(f syscall.Filetime) int64 { return int64(f.HighDateTime)<<32 | int64(f.LowDateTime) }
|
|
243
|
+
return time.Duration(ticks(kernel)+ticks(user)) * 100
|
|
244
|
+
}
|