soft-sort 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +35 -0
- data/LICENSE +5 -0
- data/README.md +146 -0
- data/bin/soft-sort +6 -0
- data/config/default_rules.yml +74 -0
- data/lib/soft_sort/classification/ai_provider.rb +17 -0
- data/lib/soft_sort/classification/classification_result.rb +6 -0
- data/lib/soft_sort/classification/content_classifier.rb +38 -0
- data/lib/soft_sort/classification/context_classifier.rb +23 -0
- data/lib/soft_sort/classification/rule_matcher.rb +41 -0
- data/lib/soft_sort/classification/sensitive_file_detector.rb +14 -0
- data/lib/soft_sort/classification/type_classifier.rb +54 -0
- data/lib/soft_sort/cli.rb +167 -0
- data/lib/soft_sort/configuration.rb +72 -0
- data/lib/soft_sort/duplicates/duplicate_detector.rb +17 -0
- data/lib/soft_sort/errors.rb +7 -0
- data/lib/soft_sort/execution/executor.rb +66 -0
- data/lib/soft_sort/execution/recovery_manager.rb +87 -0
- data/lib/soft_sort/execution/undo_manager.rb +45 -0
- data/lib/soft_sort/metadata/image_metadata.rb +24 -0
- data/lib/soft_sort/planning/organization_planner.rb +115 -0
- data/lib/soft_sort/reporting/json_reporter.rb +7 -0
- data/lib/soft_sort/reporting/preview_formatter.rb +17 -0
- data/lib/soft_sort/reporting/statistics_reporter.rb +13 -0
- data/lib/soft_sort/scanning/directory_scanner.rb +69 -0
- data/lib/soft_sort/scanning/file_metadata.rb +6 -0
- data/lib/soft_sort/storage/root_lock.rb +21 -0
- data/lib/soft_sort/storage/transaction_journal.rb +56 -0
- data/lib/soft_sort/version.rb +5 -0
- data/lib/soft_sort/watching/directory_watcher.rb +42 -0
- data/lib/soft_sort.rb +36 -0
- metadata +75 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 275107ab192d1c7a5b4a0e5e3f5a7a4b68c5549b7309e2463a2e117b3a69b3b7
|
|
4
|
+
data.tar.gz: 9c7d12b6085ce04206251509b59afc59dae3cbe3e08e88802b09017f066b9ac7
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: f751d4027048e81d6ea8964d751502ede00f8c3fe78a6eac4d24fd91fe89e5d2c738b0ae281820c8404febac9aec4ce9e6967127ab1b85b0ee1d6da38ff82123
|
|
7
|
+
data.tar.gz: '090d15e5f83d903ad64392a3b671a6dffcc6a70f7ecbde1d0bc0f19656f6feaf341124371625be266261172872c1a3671dde7ba43f41cdaefb3f1c19db761b09'
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 1.0.0 - 2026-11-13
|
|
4
|
+
|
|
5
|
+
- Fixed Ruby 3.2 development dependency compatibility by constraining Parallel to the 1.27 series.
|
|
6
|
+
- Fixed scanning on systems without birth timestamps; unavailable creation dates remain nil and automatic date folders use modification time.
|
|
7
|
+
- Excluded vendored dependencies from RuboCop while retaining its default exclusions.
|
|
8
|
+
- Expanded YAML-driven file recognition across development, documentation,
|
|
9
|
+
configuration, design, font, CAD, database, e-book, and media formats.
|
|
10
|
+
- Added bounded extensionless filename/signature recognition without execution.
|
|
11
|
+
- Added project type subfolders and recursive protection for established trees.
|
|
12
|
+
- Added regression coverage for CivicWatch-style project scripts, sensitive
|
|
13
|
+
files, unknown formats, and recursive safety.
|
|
14
|
+
- Added explicit dry-run and `organize --apply` execution.
|
|
15
|
+
- Added protected-file detection, SHA-256 pre-move validation, no-overwrite moves, and required-folder creation.
|
|
16
|
+
- Added durable JSONL transaction history and verified undo support.
|
|
17
|
+
- Added duplicate detection, statistics, and explainability commands.
|
|
18
|
+
- Added expanded safety, execution, duplicate, and undo tests.
|
|
19
|
+
- Documented intentionally disabled content, EXIF, AI, watch, and automatic recovery features.
|
|
20
|
+
- Added root-scoped locking, synchronized/fsynced JSONL journal writes, per-operation progress persistence, and filesystem reconciliation for interrupted moves.
|
|
21
|
+
- Added read-only and apply-mode `recover` command with hash-verified recovery reporting.
|
|
22
|
+
- Hardened moves and undo with same-filesystem no-replace hard-link/unlink operations and symlink rejection.
|
|
23
|
+
- Added configurable year, year/month, and year/month/day destination folders.
|
|
24
|
+
- Added opt-in bounded local content evidence for text, Markdown, CSV, JSON, and PDF files.
|
|
25
|
+
- Added explicit `skip`, `review`, and deterministic `rename` collision policies.
|
|
26
|
+
- Added advanced classification and collision regression coverage.
|
|
27
|
+
- Added polling-based, stability-aware `watch` mode with preview-only defaults.
|
|
28
|
+
- Added disabled-by-default AI provider interfaces and bounded PNG metadata support.
|
|
29
|
+
- Added GitHub Actions CI for Ruby/RSpec/RuboCop, CLI smoke tests, gem builds,
|
|
30
|
+
and macOS/Linux/Windows runner coverage.
|
|
31
|
+
- Added isolated installed-gem smoke verification and release support documentation.
|
|
32
|
+
|
|
33
|
+
## 0.1.0 - 2026-10-16
|
|
34
|
+
|
|
35
|
+
- Added read-only scanning, hybrid classification, planning, text output, and JSON reporting.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SoftSort contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files, to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and sell copies of the Software.
|
data/README.md
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# SoftSort
|
|
2
|
+
|
|
3
|
+
SoftSort is a local-first, offline Ruby CLI that scans a selected directory, explains deterministic organization decisions, and—when explicitly requested—moves approved files into generated folders. It never follows symbolic links, silently overwrites files, or deletes files.
|
|
4
|
+
|
|
5
|
+
## Requirements and installation
|
|
6
|
+
|
|
7
|
+
Ruby 3.2 or newer is required. SoftSort has no runtime service, database, cloud, or AI dependency.
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
git clone <repository>
|
|
11
|
+
cd soft-sort
|
|
12
|
+
bundle install
|
|
13
|
+
bundle exec ruby bin/soft-sort --help
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The same commands work in macOS/Linux shells and Windows PowerShell (use `ruby bin/soft-sort ...` in place of an installed executable).
|
|
17
|
+
|
|
18
|
+
## CLI usage
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
soft-sort --help
|
|
22
|
+
soft-sort version
|
|
23
|
+
soft-sort scan ~/Downloads
|
|
24
|
+
soft-sort plan ~/Downloads
|
|
25
|
+
soft-sort plan ~/Downloads --recursive
|
|
26
|
+
soft-sort plan ~/Downloads --format json
|
|
27
|
+
soft-sort plan ~/Downloads --config ./rules.yml --verbose
|
|
28
|
+
soft-sort organize ~/Downloads --apply
|
|
29
|
+
soft-sort duplicates ~/Downloads --recursive --format json
|
|
30
|
+
soft-sort history ~/Downloads --limit 20
|
|
31
|
+
soft-sort undo ~/Downloads --dry-run
|
|
32
|
+
soft-sort recover ~/Downloads
|
|
33
|
+
soft-sort recover ~/Downloads --apply
|
|
34
|
+
soft-sort stats ~/Downloads --format json
|
|
35
|
+
soft-sort explain ~/Downloads/example.pdf
|
|
36
|
+
soft-sort plan ~/Downloads --date-folders --inspect-content
|
|
37
|
+
soft-sort organize ~/Downloads --collision rename --apply
|
|
38
|
+
soft-sort watch ~/Downloads --interval 5
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
`scan` reports discovered, eligible, excluded, categorized, and sized files. `plan` reports source, proposed destination, type, context, rule, confidence, action, and warnings. `organize` previews by default and performs approved moves only with `--apply`. `--recursive` is opt-in. `--format json` emits only JSON on stdout; diagnostics go to stderr.
|
|
42
|
+
|
|
43
|
+
Recovery is read-only by default. It reconciles unfinished journal transactions against the actual source and destination files using stored SHA-256 identities. Use `recover --apply` only after reviewing the preview. Recovery refuses symlinks, missing identities, collisions, modified files, and operations outside the selected root.
|
|
44
|
+
|
|
45
|
+
Example:
|
|
46
|
+
|
|
47
|
+
```text
|
|
48
|
+
MOVE invoice_2026.pdf -> Finance/Invoices/invoice_2026.pdf (Documents, 0.95)
|
|
49
|
+
MOVE Screenshot_2026.png -> Images/Screenshots/Screenshot_2026.png (Images, 0.95)
|
|
50
|
+
REVIEW mystery.xyz -> Others/mystery.xyz (Others, 0.25)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## How classification works
|
|
54
|
+
|
|
55
|
+
Type classification is YAML-driven and covers documents/e-books, images/graphics, spreadsheets, presentations, video, audio, archives, installers, fonts, design, CAD, databases, development languages, documentation, shell scripts, and configuration. It recognizes Markdown (`md`, `markdown`), shell scripts (`sh`, `bash`, `zsh`, `fish`), development formats (`rb`, `py`, `go`, `js`, `jsx`, `ts`, `tsx`, `sql`), configuration (`json`, `yaml`, `yml`, `toml`, `xml`, `plist`, `ini`, `conf`, `env`), and common format extensions. Extensionless files can use safe filename rules or a bounded signature read for PDF, ZIP, and shebang headers; SoftSort never executes them. Unknown extensions use `Others` and require review.
|
|
56
|
+
|
|
57
|
+
Context classification tokenizes case-insensitive filenames using common separators. Rules are ordered by priority. The precedence is contextual rule, file type, then unknown fallback. Equal-priority contextual matches are marked `review` rather than chosen arbitrarily. Confidence is a deterministic evidence score, not a statistical probability.
|
|
58
|
+
|
|
59
|
+
Projects are configured with aliases and optional type destinations. For example, a `CivicWatch-deployment.sh` file can resolve to `Projects/CivicWatch/Development/Scripts/`. Recursive mode protects files already inside an established directory tree by default, so existing project source and manifests are not scattered.
|
|
60
|
+
|
|
61
|
+
Date folders are disabled by default and can be enabled with `--date-folders` or YAML. Supported formats are `year`, `year_month`, and `year_month_day`; the default date source is a reliable available timestamp, falling back to modification time. Collision handling defaults to conservative review/conflict behavior. Set `organization.collision_policy` to `rename` only when deterministic `(1)`, `(2)` suffixes are desired.
|
|
62
|
+
|
|
63
|
+
Bounded local content evidence is disabled by default. `--inspect-content` enables small, offline reads for plain text, Markdown, CSV, JSON, and PDF text; protected files are never inspected. The implementation does not execute documents, extract arbitrary archives, or upload content.
|
|
64
|
+
|
|
65
|
+
`watch` uses polling and is preview-only unless `--apply` is supplied. It waits for files to remain size/timestamp-stable before planning them and inherits all scanner, project, sensitive-file, lock, collision, and journal safeguards. Stop it with Ctrl-C. The optional AI provider interface is disabled by default (`ai.enabled: false`) and has no network implementation; deterministic classification remains the only active fallback.
|
|
66
|
+
|
|
67
|
+
Default destinations include `Images/Photos`, `Documents`, `Media/Videos`, `Media/Audio`, `Archives`, `Projects/Source`, `Installers`, and contextual destinations such as `Finance/Invoices`, `Images/Screenshots`, and `Documents/Reports`.
|
|
68
|
+
|
|
69
|
+
## Configuration
|
|
70
|
+
|
|
71
|
+
Use safe YAML with these top-level sections:
|
|
72
|
+
|
|
73
|
+
```yaml
|
|
74
|
+
version: 1
|
|
75
|
+
organization:
|
|
76
|
+
mode: hybrid
|
|
77
|
+
recursive: false
|
|
78
|
+
dry_run: true
|
|
79
|
+
safety:
|
|
80
|
+
follow_symlinks: false
|
|
81
|
+
include_hidden: false
|
|
82
|
+
categories:
|
|
83
|
+
Documents:
|
|
84
|
+
extensions: [pdf, doc, docx, txt]
|
|
85
|
+
destination: Documents
|
|
86
|
+
context_rules:
|
|
87
|
+
- name: invoices
|
|
88
|
+
keywords: [invoice, receipt]
|
|
89
|
+
destination: Finance/Invoices
|
|
90
|
+
priority: 100
|
|
91
|
+
projects:
|
|
92
|
+
civicwatch:
|
|
93
|
+
aliases: [civicwatch, civic-watch]
|
|
94
|
+
destination: Projects/CivicWatch
|
|
95
|
+
destinations:
|
|
96
|
+
Development Scripts: Projects/CivicWatch/Development/Scripts
|
|
97
|
+
Development: Projects/CivicWatch/Development
|
|
98
|
+
exclusions:
|
|
99
|
+
directories: [node_modules]
|
|
100
|
+
extensions: [part, crdownload]
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Destinations must be relative and cannot contain `..`. Safe YAML loading rejects aliases and invalid configuration. Default exclusions include hidden entries, symlinks, `.git`, `node_modules`, `.venv`, `vendor/bundle`, `__pycache__`, and incomplete-download extensions (`.part`, `.crdownload`, `.download`, `.tmp`).
|
|
104
|
+
|
|
105
|
+
## Safety guarantees and limitations
|
|
106
|
+
|
|
107
|
+
SoftSort defaults to a dry run. With `organize --apply`, it creates only required destination folders and moves approved, non-sensitive, non-conflicting files. It verifies SHA-256 identity immediately before each move, uses a same-filesystem no-replace hard-link/unlink operation, refuses existing destinations, records append-only JSONL transactions under `.soft-sort`, and supports verified undo and recovery. It never follows symbolic links, leaves the selected root, executes scanned files, unpacks archives, transmits metadata, overwrites or deletes files, or moves protected credentials by default.
|
|
108
|
+
|
|
109
|
+
Duplicate detection uses size grouping and streamed SHA-256 hashing but never deletes duplicates. Journal recovery is supported for interrupted move transactions; power-loss durability remains subject to the host filesystem's fsync guarantees. Image metadata currently provides bounded PNG dimensions only; full EXIF, HEIC/WebP capture dates, DOCX extraction, OCR, and remote/local model execution remain disabled until they can be implemented with platform-specific safety guarantees. Extension/name classification can be wrong and should be reviewed.
|
|
110
|
+
|
|
111
|
+
## Project structure
|
|
112
|
+
|
|
113
|
+
```text
|
|
114
|
+
bin/soft-sort CLI executable
|
|
115
|
+
lib/soft_sort/scanning filesystem scanning and metadata
|
|
116
|
+
lib/soft_sort/classification type/context/rule classification
|
|
117
|
+
lib/soft_sort/planning destination and conflict planning
|
|
118
|
+
lib/soft_sort/reporting text and JSON output
|
|
119
|
+
config/default_rules.yml default rules
|
|
120
|
+
spec/ automated tests with temporary fixtures
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Testing and development
|
|
124
|
+
|
|
125
|
+
```sh
|
|
126
|
+
bundle exec rspec
|
|
127
|
+
bundle exec rubocop
|
|
128
|
+
bundle exec ruby bin/soft-sort scan ./test-files
|
|
129
|
+
bundle exec ruby bin/soft-sort plan ./test-files --recursive --format json
|
|
130
|
+
gem build soft-sort.gemspec
|
|
131
|
+
ruby script/installed_gem_smoke.rb soft-sort-1.0.0.gem
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The test suite uses temporary directories rather than personal folders and covers empty/single/nested scans, hidden files, symlinks, exclusions, temporary downloads, missing roots, extension/context classification, priority ambiguity, deterministic planning, already-organized files, configuration safety, CLI output/errors, JSON validity, read-only behavior, execution, undo, recovery reconciliation, malformed journals, date folders, collision renaming, bounded content evidence, PNG metadata, and disabled AI fallback.
|
|
135
|
+
|
|
136
|
+
## Platform support and release status
|
|
137
|
+
|
|
138
|
+
SoftSort requires Ruby 3.2 or newer. The source is designed for macOS, Linux, and Windows; hosted CI is configured for Ruby 3.2 and 3.3 on Ubuntu, macOS, and Windows. Local verification has been performed on macOS Apple Silicon. Hosted CI results must be checked before claiming a platform-specific release guarantee.
|
|
139
|
+
|
|
140
|
+
Same-filesystem organization uses no-replace hard-link/unlink operations. Cross-filesystem moves are rejected rather than copied and deleted. Locking, timestamp precision, symlink behavior, hard-link permissions, and power-loss durability depend on the host filesystem. Watch mode is polling-based and preview-only unless `--apply` is supplied.
|
|
141
|
+
|
|
142
|
+
The release artifact is built with `gem build soft-sort.gemspec`. The installed-gem smoke test verifies the executable outside the source load path. No publishing workflow is included; publishing requires explicit manual authorization.
|
|
143
|
+
|
|
144
|
+
## Contributing and license
|
|
145
|
+
|
|
146
|
+
Keep the scanner and classifiers free of filesystem mutation, add focused temporary-fixture tests for changes, and run RSpec and RuboCop before submitting a change. SoftSort is released under the MIT License; see [LICENSE](LICENSE).
|
data/bin/soft-sort
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
version: 1
|
|
2
|
+
organization:
|
|
3
|
+
mode: hybrid
|
|
4
|
+
recursive: false
|
|
5
|
+
dry_run: true
|
|
6
|
+
date_folders: false
|
|
7
|
+
date_format: year_month
|
|
8
|
+
date_source: auto
|
|
9
|
+
collision_policy: skip
|
|
10
|
+
safety:
|
|
11
|
+
follow_symlinks: false
|
|
12
|
+
include_hidden: false
|
|
13
|
+
overwrite: false
|
|
14
|
+
protect_existing_project_structures: true
|
|
15
|
+
categories:
|
|
16
|
+
Images: { extensions: [jpg, jpeg, png, gif, webp, heic, svg, tiff], destination: Images/Photos }
|
|
17
|
+
Documents: { extensions: [pdf, doc, docx, txt, rtf, odt, epub, mobi, azw, azw3], destination: Documents }
|
|
18
|
+
Spreadsheets: { extensions: [xls, xlsx, csv, ods], destination: Documents/Spreadsheets }
|
|
19
|
+
Videos: { extensions: [mp4, mov, avi, mkv, webm], destination: Media/Videos }
|
|
20
|
+
Audio: { extensions: [mp3, wav, flac, aac, m4a], destination: Media/Audio }
|
|
21
|
+
Archives: { extensions: [zip, rar, 7z, tar, gz, bz2, xz, tgz, iso, cab], destination: Archives }
|
|
22
|
+
Development: { extensions: [rb, py, go, js, jsx, ts, tsx, rs, java, c, h, cpp, cs, php, swift, kt, dart, lua, r, sql], destination: Projects/Source }
|
|
23
|
+
Documentation: { extensions: [md, markdown, rst, adoc, tex], destination: Documents/Documentation }
|
|
24
|
+
Development Scripts: { extensions: [sh, bash, zsh, fish, ps1, bat, cmd], destination: Projects/Source/Scripts }
|
|
25
|
+
Configuration: { extensions: [json, yaml, yml, toml, xml, plist, ini, conf, cfg, properties, env], destination: Projects/Source/Configuration }
|
|
26
|
+
Design: { extensions: [fig, sketch, ai, eps, psd, xd, indd], destination: Design }
|
|
27
|
+
Fonts: { extensions: [ttf, otf, woff, woff2], destination: Fonts }
|
|
28
|
+
Databases: { extensions: [db, sqlite, sqlite3, sqlitedb, mdb], destination: Data/Databases }
|
|
29
|
+
CAD: { extensions: [dwg, dxf, step, stp, iges, igs, stl, obj], destination: Design/CAD }
|
|
30
|
+
Source Code: { extensions: [rb, py, js, jsx, ts, tsx, go, rs, java, html, css], destination: Projects/Source }
|
|
31
|
+
Installers: { extensions: [dmg, pkg, exe, msi, deb, rpm, appimage], destination: Installers }
|
|
32
|
+
Presentations: { extensions: [ppt, pptx, odp, key], destination: Documents/Presentations }
|
|
33
|
+
Graphics: { extensions: [svg, psd, ai, eps], destination: Images/Graphics }
|
|
34
|
+
context_rules:
|
|
35
|
+
- { name: invoices, keywords: [invoice, receipt], destination: Finance/Invoices, priority: 100 }
|
|
36
|
+
- { name: screenshots, keywords: [screenshot, screen_shot], destination: Images/Screenshots, priority: 90 }
|
|
37
|
+
- { name: project_giffy, keywords: [giffy, giffyride], destination: Projects/Giffy, priority: 80 }
|
|
38
|
+
- { name: resumes, keywords: [resume, cv], destination: Personal/Resumes, priority: 80 }
|
|
39
|
+
- { name: contracts, keywords: [contract, agreement], destination: Finance/Contracts, priority: 80 }
|
|
40
|
+
- { name: reports, keywords: [report], destination: Documents/Reports, priority: 70 }
|
|
41
|
+
projects:
|
|
42
|
+
civicwatch:
|
|
43
|
+
aliases: [civicwatch, civic-watch]
|
|
44
|
+
destination: Projects/CivicWatch
|
|
45
|
+
destinations:
|
|
46
|
+
Development: Projects/CivicWatch/Development
|
|
47
|
+
Development Scripts: Projects/CivicWatch/Development/Scripts
|
|
48
|
+
Configuration: Projects/CivicWatch/Development/Configuration
|
|
49
|
+
Documentation: Projects/CivicWatch/Documentation
|
|
50
|
+
giffy:
|
|
51
|
+
aliases: [giffy, giffyride]
|
|
52
|
+
destination: Projects/Giffy
|
|
53
|
+
destinations:
|
|
54
|
+
Development: Projects/Giffy/Development
|
|
55
|
+
Development Scripts: Projects/Giffy/Development/Scripts
|
|
56
|
+
Configuration: Projects/Giffy/Development/Configuration
|
|
57
|
+
Documentation: Projects/Giffy/Documentation
|
|
58
|
+
Images: Projects/Giffy/Images
|
|
59
|
+
Documents: Projects/Giffy/Documents
|
|
60
|
+
filename_types:
|
|
61
|
+
- { pattern: '^README(?:\.|$)', category: Documentation, destination: Documents/Documentation }
|
|
62
|
+
- { pattern: '^LICENSE(?:\.|$)', category: Documentation, destination: Documents/Documentation }
|
|
63
|
+
- { pattern: '^\.env(?:\.|$)', category: Configuration, destination: Projects/Source/Configuration }
|
|
64
|
+
- { pattern: '^(Makefile|Dockerfile|Procfile)$', category: Development Scripts, destination: Projects/Source/Scripts }
|
|
65
|
+
content_inspection:
|
|
66
|
+
enabled: false
|
|
67
|
+
max_bytes: 1048576
|
|
68
|
+
min_signals: 2
|
|
69
|
+
ai:
|
|
70
|
+
enabled: false
|
|
71
|
+
provider: none
|
|
72
|
+
exclusions:
|
|
73
|
+
directories: [.git, node_modules, .venv, vendor/bundle, __pycache__]
|
|
74
|
+
extensions: [part, crdownload, download, tmp]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
module AI
|
|
5
|
+
class DisabledProvider
|
|
6
|
+
def classify(_file)
|
|
7
|
+
nil
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
class ProviderInterface
|
|
12
|
+
def classify(_file)
|
|
13
|
+
raise NotImplementedError, 'AI providers must implement #classify'
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
class ContentClassifier
|
|
5
|
+
RULES = [
|
|
6
|
+
{ name: 'invoice_content', keywords: %w[invoice billing amount due], destination: 'Finance/Invoices', category: 'Documents' },
|
|
7
|
+
{ name: 'research_content', keywords: %w[research methodology references abstract], destination: 'Documents/Education/Research', category: 'Documents' },
|
|
8
|
+
{ name: 'contract_content', keywords: %w[agreement contract parties obligations], destination: 'Finance/Contracts', category: 'Documents' }
|
|
9
|
+
].freeze
|
|
10
|
+
|
|
11
|
+
def initialize(config)
|
|
12
|
+
settings = config.data.fetch('content_inspection', {})
|
|
13
|
+
@enabled = settings.fetch('enabled', false)
|
|
14
|
+
@max_bytes = settings.fetch('max_bytes', 1_048_576)
|
|
15
|
+
@min_signals = settings.fetch('min_signals', 2)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def classify(file)
|
|
19
|
+
return nil unless @enabled && readable_format?(file) && !file.sensitive_reason
|
|
20
|
+
|
|
21
|
+
content = File.open(file.absolute_path, 'rb') { |stream| stream.read(@max_bytes) }.to_s.force_encoding(Encoding::UTF_8).scrub.downcase
|
|
22
|
+
match = RULES.map { |rule| [rule, rule[:keywords].count { |keyword| content.include?(keyword) }] }
|
|
23
|
+
.select { |_rule, signals| signals >= @min_signals }.max_by(&:last)
|
|
24
|
+
return nil unless match
|
|
25
|
+
|
|
26
|
+
rule, signals = match
|
|
27
|
+
{ category: rule[:category], destination: rule[:destination], name: rule[:name], confidence: [0.70 + (signals * 0.05), 0.90].min.round(2) }
|
|
28
|
+
rescue SystemCallError, ArgumentError
|
|
29
|
+
nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def readable_format?(file)
|
|
35
|
+
%w[txt md markdown csv json pdf].include?(file.extension)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
class ContextClassifier
|
|
5
|
+
def initialize(config)
|
|
6
|
+
projects = config.projects.map do |name, project|
|
|
7
|
+
{ 'name' => "project_#{name}", 'keywords' => project['aliases'],
|
|
8
|
+
'destination' => project['destination'], 'project_destinations' => project['destinations'],
|
|
9
|
+
'priority' => project.fetch('priority', 85) }
|
|
10
|
+
end
|
|
11
|
+
@rules = (config.context_rules + projects).sort_by { |r| -r.fetch('priority', 0).to_i }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def matches(filename)
|
|
15
|
+
tokens = filename.downcase.scan(/[[:alnum:]]+/)
|
|
16
|
+
@rules.select do |r|
|
|
17
|
+
Array(r['keywords']).any? do |k|
|
|
18
|
+
tokens == k.to_s.downcase.scan(/[[:alnum:]]+/) || tokens.include?(k.to_s.downcase)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
class RuleMatcher
|
|
5
|
+
def initialize(config)
|
|
6
|
+
@config = config
|
|
7
|
+
@types = TypeClassifier.new(config)
|
|
8
|
+
@contexts = ContextClassifier.new(config)
|
|
9
|
+
@content = ContentClassifier.new(config)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def classify(file)
|
|
13
|
+
type, type_dest = @types.classify(file)
|
|
14
|
+
matches = @contexts.matches(file.filename)
|
|
15
|
+
if matches.any?
|
|
16
|
+
top = matches.first
|
|
17
|
+
tied = matches.take_while do |r|
|
|
18
|
+
r.fetch('priority', 0).to_i == top.fetch('priority', 0).to_i
|
|
19
|
+
end
|
|
20
|
+
destination = tied.length == 1 ? project_destination(top, type, type_dest) : type_dest
|
|
21
|
+
return ClassificationResult.new(type_category: type,
|
|
22
|
+
context_category: tied.length == 1 ? top['name'].to_s.capitalize : 'Review', destination: destination, matched_rule: tied.length == 1 ? top['name'] : 'ambiguous', confidence: tied.length == 1 ? 0.95 : 0.40, requires_review: tied.length != 1)
|
|
23
|
+
end
|
|
24
|
+
content = @content.classify(file)
|
|
25
|
+
if content
|
|
26
|
+
return ClassificationResult.new(type_category: content[:category], context_category: 'Content',
|
|
27
|
+
destination: content[:destination], matched_rule: content[:name],
|
|
28
|
+
confidence: content[:confidence], requires_review: false)
|
|
29
|
+
end
|
|
30
|
+
known = @types.known?(file)
|
|
31
|
+
ClassificationResult.new(type_category: type, context_category: nil, destination: type_dest,
|
|
32
|
+
matched_rule: known ? "file_type_#{file.extension}" : 'unknown_file', confidence: known ? 0.80 : 0.25, requires_review: !known)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def project_destination(rule, type, fallback)
|
|
36
|
+
return rule['destination'] unless rule['project_destinations'].is_a?(Hash)
|
|
37
|
+
|
|
38
|
+
rule['project_destinations'].fetch(type, fallback)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
class SensitiveFileDetector
|
|
5
|
+
PATTERNS = [
|
|
6
|
+
/(^|\.)env(?:\.|$)/i, /password|secret|credential|token|recovery[-_ ]?code/i,
|
|
7
|
+
/private[-_ ]?key|id_rsa|keystore|service[-_ ]?account/i, /\.(pem|p12|p8|key|der)$/i
|
|
8
|
+
].freeze
|
|
9
|
+
|
|
10
|
+
def self.reason(filename)
|
|
11
|
+
PATTERNS.find { |pattern| filename.match?(pattern) } && 'possible sensitive or security credential'
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
class TypeClassifier
|
|
5
|
+
def initialize(config)
|
|
6
|
+
@map = config.extension_categories
|
|
7
|
+
@filename_rules = config.data.fetch('filename_types', [])
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def classify(file)
|
|
11
|
+
return signature_match(file) || filename_match(file) if file.extension.empty? || !@map.key?(file.extension)
|
|
12
|
+
|
|
13
|
+
@map.fetch(file.extension)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def known?(file)
|
|
17
|
+
return true if @map.key?(file.extension)
|
|
18
|
+
|
|
19
|
+
return true if signature_match(file)
|
|
20
|
+
|
|
21
|
+
@filename_rules.any? do |item|
|
|
22
|
+
File.basename(file.filename).match?(Regexp.new(item['pattern'], Regexp::IGNORECASE))
|
|
23
|
+
rescue RegexpError
|
|
24
|
+
false
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def signature_match(file)
|
|
31
|
+
return unless file.absolute_path && File.file?(file.absolute_path)
|
|
32
|
+
|
|
33
|
+
header = File.open(file.absolute_path, 'rb') { |stream| stream.read(32) }
|
|
34
|
+
return %w[Documents Documents] if header&.start_with?('%PDF-')
|
|
35
|
+
return %w[Archives Archives] if header&.start_with?("PK\x03\x04")
|
|
36
|
+
return ['Development Scripts', 'Projects/Source/Scripts'] if header&.start_with?('#!')
|
|
37
|
+
|
|
38
|
+
nil
|
|
39
|
+
rescue SystemCallError
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def filename_match(file)
|
|
44
|
+
rule = @filename_rules.find do |item|
|
|
45
|
+
File.basename(file.filename).match?(Regexp.new(item['pattern'], Regexp::IGNORECASE))
|
|
46
|
+
end
|
|
47
|
+
return [rule['category'], rule['destination']] if rule
|
|
48
|
+
|
|
49
|
+
@map.fetch('', %w[Others Others])
|
|
50
|
+
rescue RegexpError
|
|
51
|
+
%w[Others Others]
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SoftSort
|
|
4
|
+
class CLI
|
|
5
|
+
COMMANDS = %w[scan plan organize duplicates history undo recover stats explain watch].freeze
|
|
6
|
+
|
|
7
|
+
def initialize(argv)
|
|
8
|
+
@argv = argv.dup
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run
|
|
12
|
+
return puts help if @argv.empty? || %w[--help -h].include?(@argv.first)
|
|
13
|
+
return puts "SoftSort #{VERSION}" if @argv.first == 'version'
|
|
14
|
+
|
|
15
|
+
command = @argv.shift
|
|
16
|
+
raise Error, "unsupported command: #{command}" unless COMMANDS.include?(command)
|
|
17
|
+
options = parse_options
|
|
18
|
+
root = @argv.shift || '.'
|
|
19
|
+
raise Error, "unexpected arguments: #{@argv.join(' ')}" unless @argv.empty?
|
|
20
|
+
config = Configuration.load(options[:config])
|
|
21
|
+
config = override_config(config, options)
|
|
22
|
+
dispatch(command, root, config, options)
|
|
23
|
+
rescue Error, OptionParser::ParseError => e
|
|
24
|
+
warn "soft-sort: #{e.message}"
|
|
25
|
+
1
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def help
|
|
29
|
+
<<~TEXT
|
|
30
|
+
Usage: soft-sort <command> DIRECTORY [options]
|
|
31
|
+
scan, plan, organize, duplicates, history, undo, recover, stats, explain, watch
|
|
32
|
+
--recursive Include subdirectories
|
|
33
|
+
--apply Execute approved organization moves
|
|
34
|
+
--dry-run Preview undo/recovery operations
|
|
35
|
+
--transaction ID Select a transaction
|
|
36
|
+
--limit NUMBER Limit history output
|
|
37
|
+
--format FORMAT Emit text or json
|
|
38
|
+
--config PATH Use YAML rules
|
|
39
|
+
--date-folders Add configured date folders to destinations
|
|
40
|
+
--inspect-content Enable bounded local content evidence
|
|
41
|
+
--collision POLICY Use skip, review, or rename collision handling
|
|
42
|
+
--interval SECONDS Polling interval for watch mode
|
|
43
|
+
TEXT
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def parse_options
|
|
49
|
+
options = { recursive: false, format: 'text', config: nil, apply: false, dry_run: false, transaction: nil, limit: nil, verbose: false,
|
|
50
|
+
date_folders: false, inspect_content: false, collision: nil, interval: 5 }
|
|
51
|
+
parser = OptionParser.new do |o|
|
|
52
|
+
o.on('--recursive') { options[:recursive] = true }
|
|
53
|
+
o.on('--apply') { options[:apply] = true }
|
|
54
|
+
o.on('--dry-run') { options[:dry_run] = true }
|
|
55
|
+
o.on('--transaction ID') { |v| options[:transaction] = v }
|
|
56
|
+
o.on('--limit NUMBER', Integer) { |v| options[:limit] = v }
|
|
57
|
+
o.on('--format FORMAT') { |v| options[:format] = v }
|
|
58
|
+
o.on('--config PATH') { |v| options[:config] = v }
|
|
59
|
+
o.on('--verbose') { options[:verbose] = true }
|
|
60
|
+
o.on('--date-folders') { options[:date_folders] = true }
|
|
61
|
+
o.on('--inspect-content') { options[:inspect_content] = true }
|
|
62
|
+
o.on('--collision POLICY') { |v| options[:collision] = v }
|
|
63
|
+
o.on('--interval SECONDS', Integer) { |v| options[:interval] = v }
|
|
64
|
+
end
|
|
65
|
+
parser.parse!(@argv)
|
|
66
|
+
raise Error, 'format must be text or json' unless %w[text json].include?(options[:format])
|
|
67
|
+
options
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def override_config(config, options)
|
|
71
|
+
return config unless options[:date_folders] || options[:inspect_content] || options[:collision]
|
|
72
|
+
|
|
73
|
+
data = Marshal.load(Marshal.dump(config.data))
|
|
74
|
+
data['organization'] ||= {}
|
|
75
|
+
data['organization']['date_folders'] = true if options[:date_folders]
|
|
76
|
+
data['organization']['collision_policy'] = options[:collision] if options[:collision]
|
|
77
|
+
data['content_inspection'] ||= {}
|
|
78
|
+
data['content_inspection']['enabled'] = true if options[:inspect_content]
|
|
79
|
+
Configuration.new(data)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def dispatch(command, root, config, options)
|
|
83
|
+
case command
|
|
84
|
+
when 'scan', 'plan', 'organize', 'stats', 'explain'
|
|
85
|
+
scan = DirectoryScanner.new(root, config: config, recursive: options[:recursive]).scan
|
|
86
|
+
items = OrganizationPlanner.new(root, config).plan(scan[:files])
|
|
87
|
+
return output(scan_report(scan, root, items), options, text: PreviewFormatter.scan(scan_report(scan, root, items))) if command == 'scan'
|
|
88
|
+
return output(plan_report(root, items, scan[:warnings]), options, text: PreviewFormatter.plan(items)) if command == 'plan'
|
|
89
|
+
if command == 'stats'
|
|
90
|
+
duplicates = DuplicateDetector.new(root).find(scan[:files])
|
|
91
|
+
return output(StatisticsReporter.build(scan, items, duplicates: duplicates), options)
|
|
92
|
+
end
|
|
93
|
+
return explain(root, items, options) if command == 'explain'
|
|
94
|
+
|
|
95
|
+
record = Executor.new(root).execute(items, apply: options[:apply])
|
|
96
|
+
output(record, options)
|
|
97
|
+
when 'duplicates'
|
|
98
|
+
scan = DirectoryScanner.new(root, config: config, recursive: options[:recursive]).scan
|
|
99
|
+
output(DuplicateDetector.new(root).find(scan[:files]), options)
|
|
100
|
+
when 'history'
|
|
101
|
+
records = TransactionJournal.new(root).all
|
|
102
|
+
records = records.last(options[:limit]) if options[:limit]
|
|
103
|
+
output(records, options)
|
|
104
|
+
when 'recover'
|
|
105
|
+
reports = if options[:apply]
|
|
106
|
+
RecoveryManager.new(root).apply(transaction_id: options[:transaction])
|
|
107
|
+
else
|
|
108
|
+
RecoveryManager.new(root).inspect(transaction_id: options[:transaction])
|
|
109
|
+
end
|
|
110
|
+
output(reports, options, text: recovery_text(reports, options[:apply]))
|
|
111
|
+
when 'undo'
|
|
112
|
+
result = UndoManager.new(root).undo(transaction_id: options[:transaction], dry_run: options[:dry_run])
|
|
113
|
+
output(result, options)
|
|
114
|
+
when 'watch'
|
|
115
|
+
watcher = DirectoryWatcher.new(root, config: config, recursive: options[:recursive], interval: options[:interval])
|
|
116
|
+
watcher.run(apply: options[:apply]) do |items, apply|
|
|
117
|
+
if apply
|
|
118
|
+
output(Executor.new(root).execute(items, apply: true), options)
|
|
119
|
+
else
|
|
120
|
+
output({ root: File.expand_path(root), files: items }, options, text: PreviewFormatter.plan(items))
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def scan_report(scan, root, items)
|
|
127
|
+
eligible = scan[:files].select(&:eligible)
|
|
128
|
+
{ root: File.expand_path(root), total: scan[:files].length, eligible: eligible.length,
|
|
129
|
+
excluded: scan[:files].length - eligible.length, bytes: eligible.sum(&:size), warnings: scan[:warnings],
|
|
130
|
+
categories: items.group_by { |item| item[:type_category] }.transform_values(&:length) }
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def plan_report(root, items, warnings)
|
|
134
|
+
{ root: File.expand_path(root), files: items, warnings: warnings }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def explain(root, items, options)
|
|
138
|
+
item = items.first
|
|
139
|
+
raise Error, 'no eligible files found' unless item
|
|
140
|
+
output(item.merge(root: File.expand_path(root)), options)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def recovery_text(reports, applied)
|
|
144
|
+
return 'No incomplete transactions require recovery.' if reports.empty?
|
|
145
|
+
|
|
146
|
+
lines = ['SoftSort Recovery Preview', '']
|
|
147
|
+
reports.each do |report|
|
|
148
|
+
lines << "Transaction: #{report['transaction']}"
|
|
149
|
+
lines << "Status: #{report['status']}"
|
|
150
|
+
lines << "Completed: #{report['completed']}"
|
|
151
|
+
lines << "Pending: #{report['pending']}"
|
|
152
|
+
lines << "Conflicts: #{report['conflicts']}"
|
|
153
|
+
lines << "Recoverable: #{report['recoverable']}"
|
|
154
|
+
end
|
|
155
|
+
lines << (applied ? 'Recovery actions applied.' : 'No files were modified.')
|
|
156
|
+
lines.join("\n")
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def output(value, options, text: nil)
|
|
160
|
+
if options[:format] == 'json'
|
|
161
|
+
puts JsonReporter.dump(value)
|
|
162
|
+
else
|
|
163
|
+
puts(text || (value.is_a?(Hash) ? value.map { |k, v| "#{k}: #{v}" }.join("\n") : value))
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|