scint 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/FEATURES.md +13 -0
- data/README.md +216 -0
- data/bin/bundler-vs-scint +233 -0
- data/bin/scint +35 -0
- data/bin/scint-io-summary +46 -0
- data/bin/scint-syscall-trace +41 -0
- data/lib/bundler/setup.rb +5 -0
- data/lib/bundler.rb +168 -0
- data/lib/scint/cache/layout.rb +131 -0
- data/lib/scint/cache/metadata_store.rb +75 -0
- data/lib/scint/cache/prewarm.rb +192 -0
- data/lib/scint/cli/add.rb +85 -0
- data/lib/scint/cli/cache.rb +316 -0
- data/lib/scint/cli/exec.rb +150 -0
- data/lib/scint/cli/install.rb +1047 -0
- data/lib/scint/cli/remove.rb +60 -0
- data/lib/scint/cli.rb +77 -0
- data/lib/scint/commands/exec.rb +17 -0
- data/lib/scint/commands/install.rb +17 -0
- data/lib/scint/credentials.rb +153 -0
- data/lib/scint/debug/io_trace.rb +218 -0
- data/lib/scint/debug/sampler.rb +138 -0
- data/lib/scint/downloader/fetcher.rb +113 -0
- data/lib/scint/downloader/pool.rb +112 -0
- data/lib/scint/errors.rb +63 -0
- data/lib/scint/fs.rb +119 -0
- data/lib/scint/gem/extractor.rb +86 -0
- data/lib/scint/gem/package.rb +62 -0
- data/lib/scint/gemfile/dependency.rb +30 -0
- data/lib/scint/gemfile/editor.rb +93 -0
- data/lib/scint/gemfile/parser.rb +275 -0
- data/lib/scint/index/cache.rb +166 -0
- data/lib/scint/index/client.rb +301 -0
- data/lib/scint/index/parser.rb +142 -0
- data/lib/scint/installer/extension_builder.rb +264 -0
- data/lib/scint/installer/linker.rb +226 -0
- data/lib/scint/installer/planner.rb +140 -0
- data/lib/scint/installer/preparer.rb +207 -0
- data/lib/scint/lockfile/parser.rb +251 -0
- data/lib/scint/lockfile/writer.rb +178 -0
- data/lib/scint/platform.rb +71 -0
- data/lib/scint/progress.rb +579 -0
- data/lib/scint/resolver/provider.rb +230 -0
- data/lib/scint/resolver/resolver.rb +249 -0
- data/lib/scint/runtime/exec.rb +141 -0
- data/lib/scint/runtime/setup.rb +45 -0
- data/lib/scint/scheduler.rb +392 -0
- data/lib/scint/source/base.rb +46 -0
- data/lib/scint/source/git.rb +92 -0
- data/lib/scint/source/path.rb +70 -0
- data/lib/scint/source/rubygems.rb +79 -0
- data/lib/scint/vendor/pub_grub/assignment.rb +20 -0
- data/lib/scint/vendor/pub_grub/basic_package_source.rb +169 -0
- data/lib/scint/vendor/pub_grub/failure_writer.rb +182 -0
- data/lib/scint/vendor/pub_grub/incompatibility.rb +150 -0
- data/lib/scint/vendor/pub_grub/package.rb +43 -0
- data/lib/scint/vendor/pub_grub/partial_solution.rb +121 -0
- data/lib/scint/vendor/pub_grub/rubygems.rb +45 -0
- data/lib/scint/vendor/pub_grub/solve_failure.rb +19 -0
- data/lib/scint/vendor/pub_grub/static_package_source.rb +61 -0
- data/lib/scint/vendor/pub_grub/strategy.rb +42 -0
- data/lib/scint/vendor/pub_grub/term.rb +105 -0
- data/lib/scint/vendor/pub_grub/version.rb +3 -0
- data/lib/scint/vendor/pub_grub/version_constraint.rb +129 -0
- data/lib/scint/vendor/pub_grub/version_range.rb +423 -0
- data/lib/scint/vendor/pub_grub/version_solver.rb +236 -0
- data/lib/scint/vendor/pub_grub/version_union.rb +178 -0
- data/lib/scint/vendor/pub_grub.rb +32 -0
- data/lib/scint/worker_pool.rb +114 -0
- data/lib/scint.rb +87 -0
- metadata +116 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 2dfc2e1d04a71bb75faa5638b47b53c72449a3ebc857da9c83286ff4d5811ba2
|
|
4
|
+
data.tar.gz: c0e0375479b2dc2522289685583b13a5dd2637b94e5999557f1a32e31943ef7f
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 54ede6af6c782642dee0fe66607830ddd78a7bda1e669e6d429188da2f0d0e97733cd014ddd80ce43c97cc9987296f699c9e274adfcef88ac7aa133369b42075
|
|
7
|
+
data.tar.gz: fb5bd293409889ddf8ce7ff16b61f51752528dc204340412f3aefca9f334221740be236fe3ba281c4325778645da03dd15a563aae101fbcaa5cbf07c91cca7a2
|
data/FEATURES.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
- we want to download everything in parallel
|
|
2
|
+
- ideally even git clones
|
|
3
|
+
- we want git clones to come to us as fast as possible, we don't need all history
|
|
4
|
+
- we want to bring inbound gems into a incoming directory for processing
|
|
5
|
+
- we focus on creating a global cache that has all the gems perfectly prepared, we don't care about re-use between different versions there, and every ruby version/arch gets its own expanded cache
|
|
6
|
+
- we attempt to do expensive operations in bulk such as mass unpacking of gems
|
|
7
|
+
- we do everything in parallel that can be parallel
|
|
8
|
+
- version number resolution works by casing the strings 1.1.1 into int64 at different orders of magnitude so that version comparisons are just int comparisons.
|
|
9
|
+
- final bundle install step is to bring all gems to ./.bundle in the most efficient operation imaginable
|
|
10
|
+
- the installation process involves compilation. We attempt to have compilation happen while its not blocking other operations, but also only one compilation at a time
|
|
11
|
+
- we have a book keeping object that governs the worker pools and that's present during each step (fetch, extract, compile, install) and recieves the tasks for each phase from the workers.
|
|
12
|
+
- i suspect that we need to fork of a worker for compilation which we then have to communicate with via some rpc format. simple "-> CALL method, <- RESULT:\n...." type line protocol through stdin/out might work well enough there.
|
|
13
|
+
|
data/README.md
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# Scint
|
|
2
|
+
|
|
3
|
+
Scint is an experimental Bundler/RubyGems replacement focused on high-throughput installs with a global cache and a fast local materialization step.
|
|
4
|
+
|
|
5
|
+
It is written in pure Ruby with no dependencies; Ruby is plenty fast for this job.
|
|
6
|
+
|
|
7
|
+
Scint is designed for full backwards compatibility with Bundler workflows:
|
|
8
|
+
|
|
9
|
+
1. It reads `Gemfile` and `Gemfile.lock`.
|
|
10
|
+
2. It writes standard `Gemfile.lock`.
|
|
11
|
+
3. It interoperates with Bundler runtime layout. For example, `BUNDLE_PATH=".bundle" bundle exec ...` is expected to work.
|
|
12
|
+
4. The intent is identical behavior with better install and execution throughput.
|
|
13
|
+
|
|
14
|
+
The core idea is:
|
|
15
|
+
|
|
16
|
+
1. Prepare artifacts once in a global cache (`~/.cache/scint`).
|
|
17
|
+
2. Materialize project-local runtime state into `.bundle/` as efficiently as possible (hardlinks where possible).
|
|
18
|
+
3. Execute work in explicit concurrent phases coordinated by a scheduler session.
|
|
19
|
+
|
|
20
|
+
## Why Scint
|
|
21
|
+
|
|
22
|
+
`scint` comes from *scintillation*: short, high-energy flashes rather than continuous glow.
|
|
23
|
+
|
|
24
|
+
That maps directly to the runtime model:
|
|
25
|
+
|
|
26
|
+
1. Event-driven scheduling.
|
|
27
|
+
2. Burst parallelism where safe.
|
|
28
|
+
3. Tight phase boundaries with clear handoffs.
|
|
29
|
+
|
|
30
|
+
## CLI
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
scint install
|
|
34
|
+
scint exec <command>
|
|
35
|
+
scint cache list
|
|
36
|
+
scint cache clear
|
|
37
|
+
scint cache dir
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Performance and IO diagnostics:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
# Ruby sampling profile (JSON)
|
|
44
|
+
SCINT_PROFILE=/tmp/scint-profile.json SCINT_PROFILE_HZ=400 scint install --force
|
|
45
|
+
|
|
46
|
+
# Ruby-level IO trace (JSONL)
|
|
47
|
+
SCINT_IO_TRACE=/tmp/scint-io.jsonl scint install --force
|
|
48
|
+
|
|
49
|
+
# Summarize high-volume IO operations for quick LLM review
|
|
50
|
+
scint-io-summary /tmp/scint-io.jsonl
|
|
51
|
+
|
|
52
|
+
# Syscall-level trace (Linux strace / macOS dtruss)
|
|
53
|
+
scint-syscall-trace /tmp/scint-sys.log -- scint install --force
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Compatibility example:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
BUNDLE_PATH=".bundle" bundle exec ruby -v
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Defaults:
|
|
63
|
+
|
|
64
|
+
1. Local install/runtime directory: `.bundle/`
|
|
65
|
+
2. Global cache root: `~/.cache/scint` (or `XDG_CACHE_HOME`)
|
|
66
|
+
|
|
67
|
+
## Install Architecture
|
|
68
|
+
|
|
69
|
+
Scint install is phase-oriented. Each phase has explicit responsibilities and feeds the next phase.
|
|
70
|
+
|
|
71
|
+
1. Parse inputs (`Gemfile`, optional `Gemfile.lock`)
|
|
72
|
+
2. Fetch source metadata (indexes, git clones)
|
|
73
|
+
3. Resolve dependency graph
|
|
74
|
+
4. Plan actions (`skip`, `link`, `download`, `build_ext`)
|
|
75
|
+
5. Download/extract/cache artifacts
|
|
76
|
+
6. Link into local `.bundle` runtime
|
|
77
|
+
7. Build native extensions after link phase is ready
|
|
78
|
+
8. Write outputs (`Gemfile.lock`, runtime lock, warnings/summary)
|
|
79
|
+
|
|
80
|
+
```mermaid
|
|
81
|
+
flowchart LR
|
|
82
|
+
A[Gemfile + Gemfile.lock] --> B[Parse + Source Discovery]
|
|
83
|
+
B --> C[Fetch Indexes / Clone Git]
|
|
84
|
+
C --> D[Resolve Graph]
|
|
85
|
+
D --> E[Planner]
|
|
86
|
+
E -->|skip| F[Already Installed]
|
|
87
|
+
E -->|link| G[Link from Extracted Cache]
|
|
88
|
+
E -->|download| H[Download .gem]
|
|
89
|
+
H --> I[Extract + Cache Metadata]
|
|
90
|
+
I --> G
|
|
91
|
+
G --> J[All Links Complete]
|
|
92
|
+
J --> K[Native Extension Build]
|
|
93
|
+
K --> L[Runtime + Lockfile Write]
|
|
94
|
+
L --> M[Done]
|
|
95
|
+
|
|
96
|
+
subgraph GlobalCache["Global Cache (~/.cache/scint)"]
|
|
97
|
+
H
|
|
98
|
+
I
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
subgraph ProjectRuntime["Project Runtime (.bundle)"]
|
|
102
|
+
G
|
|
103
|
+
K
|
|
104
|
+
L
|
|
105
|
+
end
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Scheduler as Session Object
|
|
109
|
+
|
|
110
|
+
The `Scheduler` is more than a queue: it is the install *session object*.
|
|
111
|
+
It owns global execution state and coordinates workers with phase-aware semantics.
|
|
112
|
+
|
|
113
|
+
The scheduler tracks:
|
|
114
|
+
|
|
115
|
+
1. Job graph and dependencies
|
|
116
|
+
2. Priority classes by job type
|
|
117
|
+
3. Worker pool scaling
|
|
118
|
+
4. Job state transitions (`pending`, `running`, `completed`, `failed`)
|
|
119
|
+
5. Follow-up chaining (for phase handoff)
|
|
120
|
+
6. Fail-fast abort state and error collection
|
|
121
|
+
7. Progress/stats snapshots used by reporting
|
|
122
|
+
|
|
123
|
+
Workers do not own global install strategy. They execute task payloads with context supplied by scheduler enqueuing and phase sequencing.
|
|
124
|
+
|
|
125
|
+
```mermaid
|
|
126
|
+
sequenceDiagram
|
|
127
|
+
participant U as User
|
|
128
|
+
participant C as scint install
|
|
129
|
+
participant S as Scheduler(Session)
|
|
130
|
+
participant W as WorkerPool
|
|
131
|
+
participant T as Task Worker
|
|
132
|
+
|
|
133
|
+
U->>C: scint install
|
|
134
|
+
C->>S: start(max_workers, fail_fast)
|
|
135
|
+
C->>S: enqueue(fetch_index/git/...)
|
|
136
|
+
S->>W: dispatch ready jobs
|
|
137
|
+
W->>T: execute payload
|
|
138
|
+
T-->>S: complete/fail + result
|
|
139
|
+
S-->>C: wait_for phase completion
|
|
140
|
+
C->>S: enqueue next phase (download/link/build_ext)
|
|
141
|
+
S-->>C: stats + errors + aborted?
|
|
142
|
+
C-->>U: summary + lockfile/runtime outputs
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Job Lifecycle
|
|
146
|
+
|
|
147
|
+
```mermaid
|
|
148
|
+
stateDiagram-v2
|
|
149
|
+
[*] --> pending
|
|
150
|
+
pending --> running: dependencies satisfied + worker slot available
|
|
151
|
+
running --> completed: success
|
|
152
|
+
running --> failed: exception / command failure
|
|
153
|
+
completed --> [*]
|
|
154
|
+
failed --> [*]
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Data Layout
|
|
158
|
+
|
|
159
|
+
Global cache (`~/.cache/scint`):
|
|
160
|
+
|
|
161
|
+
1. `inbound/` downloaded gem files
|
|
162
|
+
2. `extracted/` unpacked gem trees
|
|
163
|
+
3. `ext/` compiled extension cache keyed by ABI
|
|
164
|
+
4. `index/` source metadata/index cache
|
|
165
|
+
5. `git/` cached git repositories
|
|
166
|
+
|
|
167
|
+
Project-local runtime (`.bundle/`):
|
|
168
|
+
|
|
169
|
+
1. `ruby/<major.minor.0>/gems/` linked gem trees
|
|
170
|
+
2. `ruby/<major.minor.0>/specifications/` gemspecs
|
|
171
|
+
3. `ruby/<major.minor.0>/bin/` gem binstubs
|
|
172
|
+
4. `bin/` project-level wrappers
|
|
173
|
+
5. `scint.lock.marshal` runtime lock for `scint exec`
|
|
174
|
+
|
|
175
|
+
## Concurrency Model
|
|
176
|
+
|
|
177
|
+
Scint parallelizes all non-conflicting work aggressively:
|
|
178
|
+
|
|
179
|
+
1. Index fetch and git clone start early.
|
|
180
|
+
2. Downloads can chain follow-up link tasks.
|
|
181
|
+
3. Planner ordering prioritizes large downloads first to keep pipeline saturated.
|
|
182
|
+
4. Build-ext runs after link readiness to make dependencies visible.
|
|
183
|
+
5. Fail-fast mode aborts scheduling of new work after the first hard failure.
|
|
184
|
+
|
|
185
|
+
## Error Model
|
|
186
|
+
|
|
187
|
+
Scint is designed to be explicit on failure:
|
|
188
|
+
|
|
189
|
+
1. Install exits non-zero on failures.
|
|
190
|
+
2. Native build failures include full captured command output.
|
|
191
|
+
3. Final summary reports installed/failed/skipped counts.
|
|
192
|
+
4. `.gitignore` warning is emitted when `.bundle/` is not ignored.
|
|
193
|
+
|
|
194
|
+
## `scint exec` Runtime
|
|
195
|
+
|
|
196
|
+
`scint exec` sets runtime env and load paths from `scint.lock.marshal`, then `exec`s the target command.
|
|
197
|
+
|
|
198
|
+
Key behaviors:
|
|
199
|
+
|
|
200
|
+
1. Injects runtime load paths and bundler compatibility shim.
|
|
201
|
+
2. Sets `GEM_HOME`/`GEM_PATH` to the local `.bundle` runtime.
|
|
202
|
+
3. Prefers local `.bundle/bin` executables.
|
|
203
|
+
4. Rebuilds runtime lock from `Gemfile.lock` + installed gems when possible if missing.
|
|
204
|
+
|
|
205
|
+
## Aspirational Direction
|
|
206
|
+
|
|
207
|
+
The current architecture already separates phases and scheduling concerns. Planned direction is to make this even more explicit:
|
|
208
|
+
|
|
209
|
+
1. First-class session object API around scheduler state and phase transitions.
|
|
210
|
+
2. Isolated compile worker process with a simple line-protocol RPC (`CALL` / `RESULT`) for stronger fault isolation.
|
|
211
|
+
3. More deterministic bulk operations for extraction/linking.
|
|
212
|
+
4. Better per-phase telemetry for latency and saturation analysis.
|
|
213
|
+
|
|
214
|
+
## Status
|
|
215
|
+
|
|
216
|
+
Scint is experimental and optimized for architecture iteration speed. Behavior and internals may change quickly.
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Benchmark: stock bundler vs scint, cold and warm caches.
|
|
3
|
+
#
|
|
4
|
+
# Cold = fresh install dir, no caches.
|
|
5
|
+
# Warm = fresh install dir, but gem download caches are warm from the cold run.
|
|
6
|
+
#
|
|
7
|
+
# No copying needed — warm runs just get a new BUNDLE_PATH while keeping
|
|
8
|
+
# the same GEM_HOME / GEM_PATH / global cache so downloads are already cached.
|
|
9
|
+
#
|
|
10
|
+
# Creates an isolated tmp directory for all outputs. Clean up with:
|
|
11
|
+
# rm -rf /tmp/scint-bench-*
|
|
12
|
+
#
|
|
13
|
+
# Usage: ./bin/bundler-vs-scint [/path/to/project]
|
|
14
|
+
# Default project: /Users/tobi/src/github.com/Shopify/u2
|
|
15
|
+
|
|
16
|
+
set -euo pipefail
|
|
17
|
+
|
|
18
|
+
PROJECT="${1:-/Users/tobi/src/github.com/Shopify/u2}"
|
|
19
|
+
SCINT_BIN="$(cd "$(dirname "$0")/.." && pwd)/bin/scint"
|
|
20
|
+
TMPROOT=$(mktemp -d /tmp/scint-bench-XXXXXX)
|
|
21
|
+
|
|
22
|
+
# Each run gets its own BUNDLE_PATH (install destination).
|
|
23
|
+
# GEM_HOME/GEM_PATH stay shared so download caches persist between cold→warm.
|
|
24
|
+
# Scint uses XDG_CACHE_HOME for its global cache — redirect to tmpdir.
|
|
25
|
+
BUNDLER_COLD="$TMPROOT/bundler-cold"
|
|
26
|
+
BUNDLER_WARM="$TMPROOT/bundler-warm"
|
|
27
|
+
BUNDLER_GEM_HOME="$TMPROOT/bundler-gem-home"
|
|
28
|
+
SCINT_COLD="$TMPROOT/scint-cold"
|
|
29
|
+
SCINT_WARM="$TMPROOT/scint-warm"
|
|
30
|
+
SCINT_GEM_HOME="$TMPROOT/scint-gem-home"
|
|
31
|
+
SCINT_XDG_CACHE="$TMPROOT/scint-cache"
|
|
32
|
+
LOGS="$TMPROOT/logs"
|
|
33
|
+
mkdir -p "$BUNDLER_COLD" "$BUNDLER_WARM" "$BUNDLER_GEM_HOME" \
|
|
34
|
+
"$SCINT_COLD" "$SCINT_WARM" "$SCINT_GEM_HOME" "$SCINT_XDG_CACHE" "$LOGS"
|
|
35
|
+
|
|
36
|
+
echo "=== Benchmark: bundler vs scint ==="
|
|
37
|
+
echo "Project: $PROJECT"
|
|
38
|
+
echo "Tmp root: $TMPROOT"
|
|
39
|
+
echo ""
|
|
40
|
+
|
|
41
|
+
# Helper: time a command, capture output, return elapsed seconds
|
|
42
|
+
run_timed() {
|
|
43
|
+
local label="$1"
|
|
44
|
+
local logfile="$2"
|
|
45
|
+
shift 2
|
|
46
|
+
|
|
47
|
+
echo "--- $label ---"
|
|
48
|
+
local start end_time elapsed
|
|
49
|
+
start=$(ruby -e 'puts Process.clock_gettime(Process::CLOCK_MONOTONIC)')
|
|
50
|
+
|
|
51
|
+
# Run in the project directory, tee to both log and stdout
|
|
52
|
+
(cd "$PROJECT" && "$@") 2>&1 | tee "$logfile" || true
|
|
53
|
+
|
|
54
|
+
end_time=$(ruby -e 'puts Process.clock_gettime(Process::CLOCK_MONOTONIC)')
|
|
55
|
+
elapsed=$(ruby -e "puts (($end_time - $start)).round(2)")
|
|
56
|
+
echo " Elapsed: ${elapsed}s (log: $logfile)"
|
|
57
|
+
echo "$elapsed" > "${logfile}.time"
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
# ── 1. Cold bundler ──────────────────────────────────────────────
|
|
61
|
+
echo ""
|
|
62
|
+
echo "════════════════════════════════════════"
|
|
63
|
+
echo " 1/4 Bundler (cold)"
|
|
64
|
+
echo "════════════════════════════════════════"
|
|
65
|
+
SYSTEM_GEM_DIR="$(ruby -e 'puts Gem.default_dir')"
|
|
66
|
+
run_timed "bundler install (cold)" "$LOGS/bundler-cold.log" \
|
|
67
|
+
env BUNDLE_PATH="$BUNDLER_COLD" \
|
|
68
|
+
GEM_HOME="$BUNDLER_GEM_HOME" \
|
|
69
|
+
GEM_PATH="$BUNDLER_GEM_HOME:$SYSTEM_GEM_DIR" \
|
|
70
|
+
BUNDLE_DISABLE_SHARED_GEMS=1 \
|
|
71
|
+
bundle install --jobs=8
|
|
72
|
+
|
|
73
|
+
# ── 2. Warm bundler (new install dir, same gem cache) ────────────
|
|
74
|
+
echo ""
|
|
75
|
+
echo "════════════════════════════════════════"
|
|
76
|
+
echo " 2/4 Bundler (warm)"
|
|
77
|
+
echo "════════════════════════════════════════"
|
|
78
|
+
run_timed "bundler install (warm)" "$LOGS/bundler-warm.log" \
|
|
79
|
+
env BUNDLE_PATH="$BUNDLER_WARM" \
|
|
80
|
+
GEM_HOME="$BUNDLER_GEM_HOME" \
|
|
81
|
+
GEM_PATH="$BUNDLER_GEM_HOME:$SYSTEM_GEM_DIR" \
|
|
82
|
+
BUNDLE_DISABLE_SHARED_GEMS=1 \
|
|
83
|
+
bundle install --jobs=8
|
|
84
|
+
|
|
85
|
+
# ── 3. Cold scint ────────────────────────────────────────────────
|
|
86
|
+
echo ""
|
|
87
|
+
echo "════════════════════════════════════════"
|
|
88
|
+
echo " 3/4 Scint (cold)"
|
|
89
|
+
echo "════════════════════════════════════════"
|
|
90
|
+
run_timed "scint install (cold)" "$LOGS/scint-cold.log" \
|
|
91
|
+
env BUNDLE_PATH="$SCINT_COLD" \
|
|
92
|
+
GEM_HOME="$SCINT_GEM_HOME" \
|
|
93
|
+
GEM_PATH="$SCINT_GEM_HOME:$SYSTEM_GEM_DIR" \
|
|
94
|
+
XDG_CACHE_HOME="$SCINT_XDG_CACHE" \
|
|
95
|
+
"$SCINT_BIN" install --path "$SCINT_COLD"
|
|
96
|
+
|
|
97
|
+
# ── 4. Warm scint (new install dir, global cache already warm) ───
|
|
98
|
+
echo ""
|
|
99
|
+
echo "════════════════════════════════════════"
|
|
100
|
+
echo " 4/4 Scint (warm)"
|
|
101
|
+
echo "════════════════════════════════════════"
|
|
102
|
+
run_timed "scint install (warm)" "$LOGS/scint-warm.log" \
|
|
103
|
+
env BUNDLE_PATH="$SCINT_WARM" \
|
|
104
|
+
GEM_HOME="$SCINT_GEM_HOME" \
|
|
105
|
+
GEM_PATH="$SCINT_GEM_HOME:$SYSTEM_GEM_DIR" \
|
|
106
|
+
XDG_CACHE_HOME="$SCINT_XDG_CACHE" \
|
|
107
|
+
"$SCINT_BIN" install --path "$SCINT_WARM"
|
|
108
|
+
|
|
109
|
+
# ── Summary ──────────────────────────────────────────────────────
|
|
110
|
+
echo ""
|
|
111
|
+
echo "════════════════════════════════════════"
|
|
112
|
+
echo " Results"
|
|
113
|
+
echo "════════════════════════════════════════"
|
|
114
|
+
|
|
115
|
+
bc=$(cat "$LOGS/bundler-cold.log.time")
|
|
116
|
+
bw=$(cat "$LOGS/bundler-warm.log.time")
|
|
117
|
+
sc=$(cat "$LOGS/scint-cold.log.time")
|
|
118
|
+
sw=$(cat "$LOGS/scint-warm.log.time")
|
|
119
|
+
|
|
120
|
+
printf " %-20s %8ss\n" "Bundler (cold):" "$bc"
|
|
121
|
+
printf " %-20s %8ss\n" "Bundler (warm):" "$bw"
|
|
122
|
+
printf " %-20s %8ss\n" "Scint (cold):" "$sc"
|
|
123
|
+
printf " %-20s %8ss\n" "Scint (warm):" "$sw"
|
|
124
|
+
echo ""
|
|
125
|
+
|
|
126
|
+
cold_speedup=$(ruby -e "puts ($bc.to_f / $sc.to_f).round(1)")
|
|
127
|
+
warm_speedup=$(ruby -e "puts ($bw.to_f / $sw.to_f).round(1)")
|
|
128
|
+
echo " Cold speedup: ${cold_speedup}x"
|
|
129
|
+
echo " Warm speedup: ${warm_speedup}x"
|
|
130
|
+
|
|
131
|
+
# ── Directory comparison ─────────────────────────────────────────
|
|
132
|
+
echo ""
|
|
133
|
+
echo "════════════════════════════════════════"
|
|
134
|
+
echo " Directory comparison"
|
|
135
|
+
echo "════════════════════════════════════════"
|
|
136
|
+
|
|
137
|
+
DIFF_LOG="$LOGS/diff-report.txt"
|
|
138
|
+
> "$DIFF_LOG"
|
|
139
|
+
|
|
140
|
+
# Compare installed gem directories
|
|
141
|
+
bundler_gems="$BUNDLER_COLD/ruby/3.4.0/gems"
|
|
142
|
+
scint_gems="$SCINT_COLD/ruby/3.4.0/gems"
|
|
143
|
+
|
|
144
|
+
if [[ -d "$bundler_gems" && -d "$scint_gems" ]]; then
|
|
145
|
+
# Gem directories present in each
|
|
146
|
+
bundler_gem_list=$(cd "$bundler_gems" && ls -1 | sort)
|
|
147
|
+
scint_gem_list=$(cd "$scint_gems" && ls -1 | sort)
|
|
148
|
+
|
|
149
|
+
only_bundler=$(comm -23 <(echo "$bundler_gem_list") <(echo "$scint_gem_list"))
|
|
150
|
+
only_scint=$(comm -13 <(echo "$bundler_gem_list") <(echo "$scint_gem_list"))
|
|
151
|
+
in_both=$(comm -12 <(echo "$bundler_gem_list") <(echo "$scint_gem_list"))
|
|
152
|
+
|
|
153
|
+
bundler_count=$(echo "$bundler_gem_list" | wc -l | tr -d ' ')
|
|
154
|
+
scint_count=$(echo "$scint_gem_list" | wc -l | tr -d ' ')
|
|
155
|
+
both_count=$(echo "$in_both" | wc -l | tr -d ' ')
|
|
156
|
+
|
|
157
|
+
echo " Bundler gems: $bundler_count"
|
|
158
|
+
echo " Scint gems: $scint_count"
|
|
159
|
+
echo " In common: $both_count"
|
|
160
|
+
|
|
161
|
+
if [[ -n "$only_bundler" ]]; then
|
|
162
|
+
echo ""
|
|
163
|
+
echo " Only in bundler:"
|
|
164
|
+
echo "$only_bundler" | sed 's/^/ /'
|
|
165
|
+
fi
|
|
166
|
+
|
|
167
|
+
if [[ -n "$only_scint" ]]; then
|
|
168
|
+
echo ""
|
|
169
|
+
echo " Only in scint:"
|
|
170
|
+
echo "$only_scint" | sed 's/^/ /'
|
|
171
|
+
fi
|
|
172
|
+
|
|
173
|
+
# Compare gemspec counts
|
|
174
|
+
bundler_specs="$BUNDLER_COLD/ruby/3.4.0/specifications"
|
|
175
|
+
scint_specs="$SCINT_COLD/ruby/3.4.0/specifications"
|
|
176
|
+
if [[ -d "$bundler_specs" && -d "$scint_specs" ]]; then
|
|
177
|
+
bs_count=$(ls -1 "$bundler_specs" | wc -l | tr -d ' ')
|
|
178
|
+
ss_count=$(ls -1 "$scint_specs" | wc -l | tr -d ' ')
|
|
179
|
+
echo ""
|
|
180
|
+
echo " Bundler gemspecs: $bs_count"
|
|
181
|
+
echo " Scint gemspecs: $ss_count"
|
|
182
|
+
fi
|
|
183
|
+
|
|
184
|
+
# Compare bin directories
|
|
185
|
+
bundler_bin="$BUNDLER_COLD/ruby/3.4.0/bin"
|
|
186
|
+
scint_bin_dir="$SCINT_COLD/ruby/3.4.0/bin"
|
|
187
|
+
if [[ -d "$bundler_bin" && -d "$scint_bin_dir" ]]; then
|
|
188
|
+
bb_count=$(ls -1 "$bundler_bin" | wc -l | tr -d ' ')
|
|
189
|
+
sb_count=$(ls -1 "$scint_bin_dir" | wc -l | tr -d ' ')
|
|
190
|
+
echo ""
|
|
191
|
+
echo " Bundler bins: $bb_count"
|
|
192
|
+
echo " Scint bins: $sb_count"
|
|
193
|
+
|
|
194
|
+
only_b_bins=$(comm -23 <(ls -1 "$bundler_bin" | sort) <(ls -1 "$scint_bin_dir" | sort))
|
|
195
|
+
only_s_bins=$(comm -13 <(ls -1 "$bundler_bin" | sort) <(ls -1 "$scint_bin_dir" | sort))
|
|
196
|
+
if [[ -n "$only_b_bins" ]]; then
|
|
197
|
+
echo " Bins only in bundler:"
|
|
198
|
+
echo "$only_b_bins" | sed 's/^/ /'
|
|
199
|
+
fi
|
|
200
|
+
if [[ -n "$only_s_bins" ]]; then
|
|
201
|
+
echo " Bins only in scint:"
|
|
202
|
+
echo "$only_s_bins" | sed 's/^/ /'
|
|
203
|
+
fi
|
|
204
|
+
fi
|
|
205
|
+
|
|
206
|
+
# Check a few random gems for file count differences
|
|
207
|
+
echo ""
|
|
208
|
+
echo " File count spot-check (first 10 shared gems):"
|
|
209
|
+
echo "$in_both" | head -10 | while read -r gem; do
|
|
210
|
+
bf=$(find "$bundler_gems/$gem" -type f 2>/dev/null | wc -l | tr -d ' ')
|
|
211
|
+
sf=$(find "$scint_gems/$gem" -type f 2>/dev/null | wc -l | tr -d ' ')
|
|
212
|
+
if [[ "$bf" != "$sf" ]]; then
|
|
213
|
+
echo " $gem: bundler=$bf scint=$sf ← DIFF"
|
|
214
|
+
else
|
|
215
|
+
echo " $gem: $bf files ✓"
|
|
216
|
+
fi
|
|
217
|
+
done
|
|
218
|
+
|
|
219
|
+
# Disk size comparison
|
|
220
|
+
echo ""
|
|
221
|
+
bundler_size=$(du -sh "$BUNDLER_COLD" 2>/dev/null | cut -f1)
|
|
222
|
+
scint_size=$(du -sh "$SCINT_COLD" 2>/dev/null | cut -f1)
|
|
223
|
+
echo " Bundler total size: $bundler_size"
|
|
224
|
+
echo " Scint total size: $scint_size"
|
|
225
|
+
else
|
|
226
|
+
echo " ⚠ Could not find gem directories to compare"
|
|
227
|
+
echo " Bundler: $bundler_gems (exists: $(test -d "$bundler_gems" && echo yes || echo no))"
|
|
228
|
+
echo " Scint: $scint_gems (exists: $(test -d "$scint_gems" && echo yes || echo no))"
|
|
229
|
+
fi
|
|
230
|
+
|
|
231
|
+
echo ""
|
|
232
|
+
echo "All logs and data in: $TMPROOT"
|
|
233
|
+
echo "Clean up: rm -rf $TMPROOT"
|
data/bin/scint
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# Ensure lib is on the load path
|
|
5
|
+
$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
|
|
6
|
+
|
|
7
|
+
require "scint"
|
|
8
|
+
|
|
9
|
+
if ENV["SCINT_IO_TRACE"] && !ENV["SCINT_IO_TRACE"].empty?
|
|
10
|
+
require "scint/debug/io_trace"
|
|
11
|
+
Scint::Debug::IOTrace.enable!(ENV["SCINT_IO_TRACE"])
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
profiler = nil
|
|
15
|
+
if ENV["SCINT_PROFILE"] && !ENV["SCINT_PROFILE"].empty?
|
|
16
|
+
require "scint/debug/sampler"
|
|
17
|
+
hz = ENV.fetch("SCINT_PROFILE_HZ", "250").to_i
|
|
18
|
+
depth = ENV.fetch("SCINT_PROFILE_DEPTH", "40").to_i
|
|
19
|
+
profiler = Scint::Debug::Sampler.new(path: ENV["SCINT_PROFILE"], hz: hz, max_depth: depth)
|
|
20
|
+
profiler.start
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
require "scint/cli"
|
|
24
|
+
|
|
25
|
+
exit_code = 1
|
|
26
|
+
begin
|
|
27
|
+
exit_code = Scint::CLI.run(ARGV)
|
|
28
|
+
ensure
|
|
29
|
+
profiler&.stop(exit_code: exit_code)
|
|
30
|
+
if defined?(Scint::Debug::IOTrace)
|
|
31
|
+
Scint::Debug::IOTrace.disable!
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
exit(exit_code)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
path = ARGV[0]
|
|
7
|
+
unless path && !path.empty?
|
|
8
|
+
warn "Usage: scint-io-summary TRACE.jsonl"
|
|
9
|
+
exit 2
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
op_counts = Hash.new(0)
|
|
13
|
+
op_path_counts = Hash.new(0)
|
|
14
|
+
line_count = 0
|
|
15
|
+
|
|
16
|
+
File.foreach(path) do |line|
|
|
17
|
+
begin
|
|
18
|
+
row = JSON.parse(line)
|
|
19
|
+
rescue JSON::ParserError
|
|
20
|
+
next
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
line_count += 1
|
|
24
|
+
op = row["op"] || "(unknown)"
|
|
25
|
+
op_counts[op] += 1
|
|
26
|
+
|
|
27
|
+
args = row.dig("data", "args")
|
|
28
|
+
first_arg = args.is_a?(Array) ? args[0] : nil
|
|
29
|
+
if first_arg.is_a?(String) && !first_arg.empty?
|
|
30
|
+
op_path_counts[[op, first_arg]] += 1
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
puts "trace_file=#{path}"
|
|
35
|
+
puts "total_rows=#{line_count}"
|
|
36
|
+
puts
|
|
37
|
+
puts "Top operations:"
|
|
38
|
+
op_counts.sort_by { |_k, v| -v }.first(30).each do |op, count|
|
|
39
|
+
puts "#{count}\t#{op}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
puts
|
|
43
|
+
puts "Top repeated op+path:"
|
|
44
|
+
op_path_counts.sort_by { |_k, v| -v }.first(50).each do |(op, p), count|
|
|
45
|
+
puts "#{count}\t#{op}\t#{p}"
|
|
46
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "rbconfig"
|
|
5
|
+
|
|
6
|
+
def usage
|
|
7
|
+
warn "Usage: scint-syscall-trace LOGFILE -- CMD [ARGS...]"
|
|
8
|
+
warn "Example: scint-syscall-trace /tmp/scint-sys.log -- scint install"
|
|
9
|
+
exit 2
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
sep = ARGV.index("--")
|
|
13
|
+
usage unless sep && sep.positive?
|
|
14
|
+
|
|
15
|
+
logfile = ARGV[0]
|
|
16
|
+
cmd = ARGV[(sep + 1)..]
|
|
17
|
+
usage if logfile.nil? || logfile.empty? || cmd.nil? || cmd.empty?
|
|
18
|
+
|
|
19
|
+
if system("command -v strace >/dev/null 2>&1")
|
|
20
|
+
exec("strace", "-ff", "-tt", "-s", "256", "-e", "trace=file,process", "-o", logfile, *cmd)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
if system("command -v dtruss >/dev/null 2>&1")
|
|
24
|
+
if Process.uid != 0
|
|
25
|
+
warn "dtruss usually requires root on macOS. Re-run with sudo:"
|
|
26
|
+
warn " sudo #{File.expand_path($PROGRAM_NAME)} #{logfile} -- #{cmd.join(" ")}"
|
|
27
|
+
exit 2
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
exec(
|
|
31
|
+
"dtruss",
|
|
32
|
+
"-f",
|
|
33
|
+
"-t", "open,open_nocancel,stat64,lstat64,access,rename,unlink,mkdir,rmdir,read,write,pread,pwrite,clonefile",
|
|
34
|
+
"-o", logfile,
|
|
35
|
+
"--",
|
|
36
|
+
*cmd
|
|
37
|
+
)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
warn "No supported syscall tracer found (expected strace or dtruss)."
|
|
41
|
+
exit 2
|