woods 1.6.1 → 2.0.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2035 -0
- data/CONTRIBUTING.md +253 -87
- data/README.md +161 -513
- data/SECURITY.md +92 -0
- data/assets/woods-wordmark-white-with-bg.png +0 -0
- data/docs/AGENT_GUIDE.md +204 -0
- data/docs/AGENT_SETUP.md +205 -0
- data/docs/BACKEND_MATRIX.md +470 -0
- data/docs/CONFIGURATION_REFERENCE.md +655 -0
- data/docs/CONSOLE_MCP_SETUP.md +829 -0
- data/docs/DOCKER_SETUP.md +454 -0
- data/docs/EMBEDDING_MODELS.md +136 -0
- data/docs/EVALUATION.md +91 -0
- data/docs/EXTRACTOR_REFERENCE.md +765 -0
- data/docs/FAQ.md +544 -0
- data/docs/GETTING_STARTED.md +183 -0
- data/docs/INCREMENTAL_EXTRACTION.md +455 -0
- data/docs/INTERNALS.md +418 -0
- data/docs/MCP_HTTP_TRANSPORT.md +144 -0
- data/docs/MCP_SERVERS.md +231 -0
- data/docs/MCP_TOOL_COOKBOOK.md +987 -0
- data/docs/MCP_WORKTREE_SETUP.md +127 -0
- data/docs/NOTION_INTEGRATION.md +283 -0
- data/docs/OBSIDIAN_INTEGRATION.md +170 -0
- data/docs/PUBLISHED_INDEX.md +213 -0
- data/docs/README.md +94 -0
- data/docs/RETRIEVAL_GUIDE.md +267 -0
- data/docs/TOKEN_BENCHMARK.md +68 -0
- data/docs/TROUBLESHOOTING.md +841 -0
- data/docs/UNBLOCKED_INTEGRATION.md +279 -0
- data/docs/UPGRADING_TO_2.md +321 -0
- data/docs/WATCH_DAEMON.md +667 -0
- data/docs/WHY_WOODS.md +219 -0
- data/exe/woods-console +40 -4
- data/exe/woods-console-mcp +21 -35
- data/exe/woods-mcp +20 -7
- data/exe/woods-mcp-http +80 -11
- data/exe/woods-mcp-start +57 -52
- data/lib/generators/woods/install_generator.rb +6 -5
- data/lib/generators/woods/pgvector_generator.rb +6 -3
- data/lib/generators/woods/templates/add_pgvector_to_woods.rb.erb +29 -9
- data/lib/generators/woods/templates/create_woods_tables.rb.erb +5 -1
- data/lib/generators/woods/templates/woods.rb.tt +49 -28
- data/lib/tasks/woods.rake +622 -168
- data/lib/tasks/woods_checks.rake +107 -0
- data/lib/tasks/woods_evaluation.rake +164 -80
- data/lib/woods/ast/call_site_extractor.rb +6 -15
- data/lib/woods/ast/method_extractor.rb +19 -9
- data/lib/woods/ast/parser.rb +54 -8
- data/lib/woods/atomic_file.rb +171 -2
- data/lib/woods/builder.rb +310 -22
- data/lib/woods/cache/cache_middleware.rb +7 -2
- data/lib/woods/cache/cache_store.rb +9 -1
- data/lib/woods/cache/solid_cache_store.rb +6 -4
- data/lib/woods/change_set.rb +88 -0
- data/lib/woods/checks/generation_resolution.rb +34 -0
- data/lib/woods/checks/moved_messages.rb +186 -0
- data/lib/woods/chunking/semantic_chunker.rb +160 -18
- data/lib/woods/console/audit_logger.rb +12 -3
- data/lib/woods/console/bridge_protocol.rb +3 -16
- data/lib/woods/console/connection_manager.rb +51 -136
- data/lib/woods/console/dispatch_pipeline.rb +42 -12
- data/lib/woods/console/embedded_executor.rb +806 -149
- data/lib/woods/console/eval_guard.rb +27 -20
- data/lib/woods/console/input_contract.rb +78 -0
- data/lib/woods/console/model_validator.rb +29 -1
- data/lib/woods/console/rack_middleware.rb +65 -42
- data/lib/woods/console/redactor.rb +26 -8
- data/lib/woods/console/safe_context.rb +58 -10
- data/lib/woods/console/scope_predicate_parser.rb +41 -0
- data/lib/woods/console/server.rb +119 -247
- data/lib/woods/console/sql_noise_stripper.rb +125 -16
- data/lib/woods/console/sql_table_scanner.rb +82 -22
- data/lib/woods/console/sql_validator.rb +459 -29
- data/lib/woods/console/table_gate.rb +2 -2
- data/lib/woods/console/tool_specs.rb +463 -90
- data/lib/woods/console/tools/tier1.rb +1 -5
- data/lib/woods/console/tools/tier4.rb +18 -9
- data/lib/woods/coordination/lock_heartbeat.rb +103 -0
- data/lib/woods/coordination/pipeline_lock.rb +263 -53
- data/lib/woods/db/migrations/007_typed_snapshot_units.rb +45 -0
- data/lib/woods/db/migrator.rb +3 -9
- data/lib/woods/db/schema_version.rb +47 -2
- data/lib/woods/dependency_graph.rb +898 -64
- data/lib/woods/embedding/fake.rb +138 -0
- data/lib/woods/embedding/indexer.rb +832 -40
- data/lib/woods/embedding/openai.rb +77 -19
- data/lib/woods/embedding/provider.rb +189 -11
- data/lib/woods/embedding/text_preparer.rb +1 -1
- data/lib/woods/embedding/token_counter.rb +0 -7
- data/lib/woods/evaluation/ablation_agent_payload.rb +38 -0
- data/lib/woods/evaluation/ablation_executor.rb +67 -0
- data/lib/woods/evaluation/ablation_provenance.rb +38 -0
- data/lib/woods/evaluation/ablation_report_writer.rb +43 -0
- data/lib/woods/evaluation/ablation_runner.rb +173 -0
- data/lib/woods/evaluation/ablation_summary.rb +65 -0
- data/lib/woods/evaluation/ablation_task.rb +66 -0
- data/lib/woods/evaluation/ablation_task_set.rb +77 -0
- data/lib/woods/evaluation/ablation_timed_executor.rb +91 -0
- data/lib/woods/evaluation/ablation_worktree.rb +71 -0
- data/lib/woods/evaluation/baseline.rb +60 -0
- data/lib/woods/evaluation/baseline_runner.rb +11 -3
- data/lib/woods/evaluation/evaluator.rb +41 -8
- data/lib/woods/evaluation/query_set.rb +79 -13
- data/lib/woods/evaluation/report_generator.rb +20 -1
- data/lib/woods/export/unit_facts.rb +0 -11
- data/lib/woods/extracted_unit.rb +22 -63
- data/lib/woods/extractor.rb +2783 -238
- data/lib/woods/extractors/action_cable_extractor.rb +9 -4
- data/lib/woods/extractors/ast_source_extraction.rb +20 -2
- data/lib/woods/extractors/caching_extractor.rb +46 -12
- data/lib/woods/extractors/callback_analyzer.rb +39 -9
- data/lib/woods/extractors/component_discovery.rb +123 -0
- data/lib/woods/extractors/concern_extractor.rb +17 -3
- data/lib/woods/extractors/controller_extractor.rb +389 -29
- data/lib/woods/extractors/decorator_extractor.rb +7 -14
- data/lib/woods/extractors/engine_extractor.rb +53 -8
- data/lib/woods/extractors/event_extractor.rb +55 -4
- data/lib/woods/extractors/factory_extractor.rb +49 -11
- data/lib/woods/extractors/graphql_extractor.rb +162 -66
- data/lib/woods/extractors/i18n_extractor.rb +6 -1
- data/lib/woods/extractors/job_extractor.rb +51 -21
- data/lib/woods/extractors/lib_extractor.rb +23 -17
- data/lib/woods/extractors/line_neutralizer.rb +171 -0
- data/lib/woods/extractors/mailer_extractor.rb +9 -1
- data/lib/woods/extractors/manager_extractor.rb +19 -2
- data/lib/woods/extractors/migration_extractor.rb +22 -11
- data/lib/woods/extractors/model_extractor.rb +292 -57
- data/lib/woods/extractors/package_extractor.rb +154 -0
- data/lib/woods/extractors/phlex_extractor.rb +18 -3
- data/lib/woods/extractors/policy_extractor.rb +6 -5
- data/lib/woods/extractors/poro_extractor.rb +13 -14
- data/lib/woods/extractors/pundit_extractor.rb +3 -3
- data/lib/woods/extractors/rails_source_extractor.rb +24 -7
- data/lib/woods/extractors/rake_task_extractor.rb +158 -30
- data/lib/woods/extractors/reference_patterns.rb +38 -0
- data/lib/woods/extractors/route_extractor.rb +58 -2
- data/lib/woods/extractors/scheduled_job_extractor.rb +51 -35
- data/lib/woods/extractors/serializer_extractor.rb +3 -4
- data/lib/woods/extractors/service_extractor.rb +11 -1
- data/lib/woods/extractors/shared_dependency_scanner.rb +24 -34
- data/lib/woods/extractors/shared_utility_methods.rb +36 -6
- data/lib/woods/extractors/source_nesting.rb +560 -0
- data/lib/woods/extractors/state_machine_extractor.rb +30 -18
- data/lib/woods/extractors/test_mapping_extractor.rb +26 -9
- data/lib/woods/extractors/view_component_extractor.rb +28 -3
- data/lib/woods/extractors/view_engines/erb.rb +17 -3
- data/lib/woods/feedback/gap_detector.rb +9 -3
- data/lib/woods/feedback/store.rb +7 -1
- data/lib/woods/filename_utils.rb +29 -1
- data/lib/woods/flow_analysis/operation_extractor.rb +22 -10
- data/lib/woods/flow_assembler.rb +147 -26
- data/lib/woods/flow_document.rb +1 -0
- data/lib/woods/flow_precomputer.rb +175 -22
- data/lib/woods/gem_mapper.rb +285 -0
- data/lib/woods/generation.rb +185 -0
- data/lib/woods/git_command.rb +38 -0
- data/lib/woods/git_provenance.rb +16 -2
- data/lib/woods/graph_analyzer.rb +564 -87
- data/lib/woods/index_artifact.rb +93 -23
- data/lib/woods/mcp/bearer_auth.rb +102 -13
- data/lib/woods/mcp/bootstrap_state.rb +77 -0
- data/lib/woods/mcp/bootstrapper.rb +582 -77
- data/lib/woods/mcp/config_resolver.rb +66 -6
- data/lib/woods/mcp/errors.rb +60 -0
- data/lib/woods/mcp/index_reader.rb +836 -117
- data/lib/woods/mcp/index_reader_pinning.rb +78 -0
- data/lib/woods/mcp/origin_guard.rb +66 -7
- data/lib/woods/mcp/protocol_policy.rb +98 -0
- data/lib/woods/mcp/provider_probe.rb +45 -6
- data/lib/woods/mcp/renderers/markdown_renderer.rb +72 -4
- data/lib/woods/mcp/renderers/plain_renderer.rb +54 -6
- data/lib/woods/mcp/server.rb +898 -152
- data/lib/woods/mcp/tasks/extension.rb +196 -0
- data/lib/woods/mcp/tasks/request_capture.rb +45 -0
- data/lib/woods/mcp/tasks/store.rb +518 -0
- data/lib/woods/mcp/tool_contract.rb +171 -0
- data/lib/woods/mcp/tool_response_renderer.rb +7 -0
- data/lib/woods/model_name_cache.rb +19 -1
- data/lib/woods/notion/client.rb +132 -36
- data/lib/woods/notion/exporter.rb +456 -61
- data/lib/woods/notion/mappers/column_mapper.rb +34 -5
- data/lib/woods/notion/mappers/migration_mapper.rb +32 -8
- data/lib/woods/notion/mappers/model_mapper.rb +21 -6
- data/lib/woods/notion/mappers/shared.rb +45 -3
- data/lib/woods/notion/sync_manifest.rb +258 -0
- data/lib/woods/obsidian/errors.rb +6 -0
- data/lib/woods/obsidian/name_mapper.rb +40 -24
- data/lib/woods/obsidian/vault_exporter.rb +103 -36
- data/lib/woods/operator/pipeline_guard.rb +118 -21
- data/lib/woods/operator/status_reporter.rb +20 -3
- data/lib/woods/path_dispatcher.rb +276 -0
- data/lib/woods/payload_store.rb +236 -0
- data/lib/woods/published_index/edge_shaper.rb +61 -0
- data/lib/woods/published_index/generation_catalog.rb +72 -0
- data/lib/woods/published_index/typed_unit_reader.rb +48 -0
- data/lib/woods/published_index.rb +287 -0
- data/lib/woods/railtie.rb +69 -30
- data/lib/woods/railtie_support.rb +167 -0
- data/lib/woods/release.rb +12 -0
- data/lib/woods/reload_policy.rb +206 -0
- data/lib/woods/resilience/circuit_breaker.rb +47 -8
- data/lib/woods/resilience/index_validator.rb +296 -10
- data/lib/woods/resilience/retryable_provider.rb +71 -6
- data/lib/woods/resolved_config.rb +55 -11
- data/lib/woods/retrieval/context_assembler.rb +132 -40
- data/lib/woods/retrieval/query_classifier.rb +26 -8
- data/lib/woods/retrieval/ranker.rb +193 -28
- data/lib/woods/retrieval/search_executor.rb +206 -39
- data/lib/woods/retriever.rb +317 -71
- data/lib/woods/retry_after.rb +22 -2
- data/lib/woods/ruby_analyzer/class_analyzer.rb +10 -14
- data/lib/woods/ruby_analyzer/fqn_builder.rb +2 -0
- data/lib/woods/ruby_analyzer/mermaid_renderer.rb +14 -4
- data/lib/woods/ruby_analyzer/method_analyzer.rb +1 -1
- data/lib/woods/ruby_analyzer/trace_enricher.rb +3 -0
- data/lib/woods/ruby_analyzer.rb +21 -5
- data/lib/woods/session_tracer/file_store.rb +138 -19
- data/lib/woods/session_tracer/middleware.rb +1 -2
- data/lib/woods/session_tracer/redis_store.rb +122 -12
- data/lib/woods/session_tracer/session_flow_assembler.rb +57 -17
- data/lib/woods/session_tracer/session_flow_document.rb +56 -14
- data/lib/woods/session_tracer/solid_cache_coordination.rb +192 -0
- data/lib/woods/session_tracer/solid_cache_store.rb +560 -91
- data/lib/woods/session_tracer/store.rb +14 -1
- data/lib/woods/storage/metadata_store.rb +230 -26
- data/lib/woods/storage/pgvector.rb +180 -22
- data/lib/woods/storage/qdrant.rb +367 -41
- data/lib/woods/storage/snapshotter/metadata.rb +79 -16
- data/lib/woods/storage/snapshotter/vector.rb +128 -17
- data/lib/woods/storage/snapshotter.rb +23 -5
- data/lib/woods/storage/vector_store.rb +49 -8
- data/lib/woods/storage_identity.rb +28 -0
- data/lib/woods/tasks.rb +53 -2
- data/lib/woods/temporal/json_snapshot_store.rb +112 -42
- data/lib/woods/temporal/snapshot_store.rb +139 -42
- data/lib/woods/unblocked/client.rb +119 -17
- data/lib/woods/unblocked/document_builder.rb +34 -2
- data/lib/woods/unblocked/exporter.rb +63 -27
- data/lib/woods/unblocked/rate_limiter.rb +23 -9
- data/lib/woods/unblocked/sync_manifest.rb +16 -8
- data/lib/woods/update_check.rb +24 -1
- data/lib/woods/util/uuid5.rb +124 -0
- data/lib/woods/version.rb +1 -1
- data/lib/woods/watch/daemon.rb +1345 -0
- data/lib/woods/watch/listen_watcher.rb +81 -0
- data/lib/woods/watch/polling_watcher.rb +137 -0
- data/lib/woods/watch/status.rb +169 -0
- data/lib/woods/watch/tree_scan.rb +163 -0
- data/lib/woods/watch/watcher.rb +100 -0
- data/lib/woods.rb +138 -9
- data/plugin/.claude-plugin/plugin.json +18 -0
- data/plugin/hooks/hooks.json +29 -0
- data/plugin/hooks/woods-post-edit.sh +226 -0
- data/plugin/hooks/woods-session-start.sh +77 -0
- data/plugin/skills/woods-agent-enable/SKILL.md +51 -0
- data/plugin/skills/woods-diagnose/SKILL.md +75 -0
- data/plugin/skills/woods-investigate/SKILL.md +39 -0
- data/plugin/skills/woods-mcp-config/SKILL.md +101 -0
- data/plugin/skills/woods-setup/SKILL.md +99 -0
- metadata +134 -23
- data/lib/woods/console/adapters/cache_adapter.rb +0 -58
- data/lib/woods/console/adapters/good_job_adapter.rb +0 -33
- data/lib/woods/console/adapters/job_adapter.rb +0 -74
- data/lib/woods/console/adapters/sidekiq_adapter.rb +0 -33
- data/lib/woods/console/adapters/solid_queue_adapter.rb +0 -33
- data/lib/woods/console/bridge.rb +0 -210
- data/lib/woods/formatting/claude_adapter.rb +0 -98
- data/lib/woods/formatting/generic_adapter.rb +0 -56
- data/lib/woods/formatting/gpt_adapter.rb +0 -64
- data/lib/woods/notion/mapper.rb +0 -40
- data/lib/woods/observability/health_check.rb +0 -79
- data/lib/woods/observability/instrumentation.rb +0 -34
|
@@ -2,8 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
require 'woods'
|
|
4
4
|
require_relative 'client'
|
|
5
|
-
require_relative '
|
|
5
|
+
require_relative 'mappers/shared'
|
|
6
|
+
require_relative 'mappers/model_mapper'
|
|
7
|
+
require_relative 'mappers/column_mapper'
|
|
8
|
+
require_relative 'mappers/migration_mapper'
|
|
6
9
|
require_relative 'rate_limiter'
|
|
10
|
+
require_relative 'sync_manifest'
|
|
7
11
|
|
|
8
12
|
module Woods
|
|
9
13
|
module Notion
|
|
@@ -13,18 +17,60 @@ module Woods
|
|
|
13
17
|
# to Notion page properties, and pushes via the Notion API. All syncs are idempotent —
|
|
14
18
|
# existing pages are updated, new pages are created.
|
|
15
19
|
#
|
|
20
|
+
# Syncs are incremental (#207 / B-095): a {SyncManifest} persisted under
|
|
21
|
+
# the index directory records each page's Notion page_id and a content
|
|
22
|
+
# hash of its mapped properties, keyed by the page's qualified title
|
|
23
|
+
# (#149). An unchanged page is skipped with zero API calls; a changed
|
|
24
|
+
# page is PATCHed directly by its cached page_id (one call, no
|
|
25
|
+
# find-by-title query); only a manifest miss pays the full lookup /
|
|
26
|
+
# legacy-adoption / create path. A cold manifest (first run, deleted
|
|
27
|
+
# file) therefore degrades to exactly the pre-manifest behavior, plus
|
|
28
|
+
# manifest writes. Set +WOODS_NOTION_FORCE=1+ (or pass
|
|
29
|
+
# +force_full: true+) to ignore the manifest for one run.
|
|
30
|
+
#
|
|
16
31
|
# @example
|
|
17
32
|
# exporter = Exporter.new(index_dir: "tmp/woods")
|
|
18
33
|
# stats = exporter.sync_all
|
|
19
|
-
# # => { data_models: 10, columns: 45, errors: [] }
|
|
34
|
+
# # => { data_models: 10, columns: 45, skipped: 0, errors: [] }
|
|
20
35
|
#
|
|
21
|
-
|
|
36
|
+
# rubocop:disable-next Metrics/ClassLength
|
|
37
|
+
class Exporter
|
|
38
|
+
MAX_ERRORS = 100
|
|
39
|
+
|
|
40
|
+
# Manifest scope for the Data Models database.
|
|
41
|
+
SCOPE_DATA_MODELS = 'data_models'
|
|
42
|
+
# Manifest scope for the Columns database.
|
|
43
|
+
SCOPE_COLUMNS = 'columns'
|
|
44
|
+
# Manifest file name, stored under the index/output directory
|
|
45
|
+
# (mirrors Unblocked's +unblocked_sync_manifest.json+).
|
|
46
|
+
MANIFEST_FILENAME = 'notion_sync_manifest.json'
|
|
47
|
+
# Escape hatch: set to 1/true/yes to ignore the manifest for one run
|
|
48
|
+
# (the Notion counterpart of +UNBLOCKED_FORCE_FULL_SYNC+). Read here
|
|
49
|
+
# rather than in the rake task so the MCP notion tool and embedded
|
|
50
|
+
# callers honor it too.
|
|
51
|
+
FORCE_ENV_VAR = 'WOODS_NOTION_FORCE'
|
|
52
|
+
|
|
53
|
+
# Client errors that mean a cached page is gone (deleted or archived
|
|
54
|
+
# behind the manifest), so the sync should self-heal by recreating it.
|
|
55
|
+
# {Client} raises untyped +Woods::Error+ with the Notion status code
|
|
56
|
+
# baked into the message ("Notion API error 404: ..."), so this is a
|
|
57
|
+
# message-level match: 404/410 (page gone) or an archived-page
|
|
58
|
+
# complaint (Notion answers 400 "Can't update a page that is
|
|
59
|
+
# archived..." for trashed pages).
|
|
60
|
+
STALE_PAGE_ERROR = /\ANotion API error (?:404|410)\b|archiv/i
|
|
61
|
+
|
|
22
62
|
# @param index_dir [String] Path to extraction output directory
|
|
23
63
|
# @param config [Configuration] Woods configuration (default: global config)
|
|
24
64
|
# @param client [Client, nil] Notion API client (auto-created from config if nil)
|
|
25
65
|
# @param reader [Object, nil] IndexReader instance (auto-created from index_dir if nil)
|
|
66
|
+
# @param manifest [SyncManifest, nil] Sync manifest (auto-created under index_dir if nil)
|
|
67
|
+
# @param force_full [Boolean, nil] Ignore the manifest for this run —
|
|
68
|
+
# every page goes through the full find-by-title path (results are
|
|
69
|
+
# still recorded). Defaults to the {FORCE_ENV_VAR} env flag.
|
|
26
70
|
# @raise [ConfigurationError] if notion_api_token is not configured
|
|
27
|
-
|
|
71
|
+
# rubocop:disable-next Metrics/ParameterLists -- injectable collaborators, mirrors Unblocked::Exporter
|
|
72
|
+
def initialize(index_dir:, config: Woods.configuration, client: nil, reader: nil,
|
|
73
|
+
manifest: nil, force_full: nil)
|
|
28
74
|
# A non-blank NOTION_API_TOKEN overrides the configured token
|
|
29
75
|
# (documented contract; also what the MCP notion_wired? gate keys on).
|
|
30
76
|
# Resolve via the shared Woods.resolve_notion_token so the exporter,
|
|
@@ -37,61 +83,104 @@ module Woods
|
|
|
37
83
|
@database_ids = config.notion_database_ids || {}
|
|
38
84
|
@client = client || Client.new(api_token: api_token)
|
|
39
85
|
@reader = reader || build_reader(index_dir)
|
|
86
|
+
@manifest = manifest || build_manifest(index_dir)
|
|
87
|
+
@force_full = force_full.nil? ? env_force? : force_full
|
|
40
88
|
@page_id_cache = {}
|
|
41
89
|
end
|
|
42
90
|
|
|
43
|
-
# Sync all configured databases. Idempotent — safe to re-run
|
|
91
|
+
# Sync all configured databases. Idempotent — safe to re-run; a re-run
|
|
92
|
+
# against an unchanged index issues zero API calls (see class docs).
|
|
44
93
|
#
|
|
45
|
-
# @return [Hash] { data_models: Integer, columns: Integer,
|
|
94
|
+
# @return [Hash] { data_models: Integer, columns: Integer,
|
|
95
|
+
# skipped: Integer, errors: Array<String> }
|
|
46
96
|
def sync_all
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
97
|
+
with_pinned_index do
|
|
98
|
+
model_stats = @database_ids[:data_models] ? sync_data_models : empty_stats
|
|
99
|
+
warn_columns_without_data_models if @database_ids[:columns] && !@database_ids[:data_models]
|
|
100
|
+
column_stats = @database_ids[:columns] ? sync_columns : empty_stats
|
|
101
|
+
|
|
102
|
+
all_errors = model_stats[:errors] + column_stats[:errors]
|
|
103
|
+
|
|
104
|
+
{
|
|
105
|
+
data_models: model_stats[:synced],
|
|
106
|
+
columns: column_stats[:synced],
|
|
107
|
+
skipped: model_stats[:skipped] + column_stats[:skipped],
|
|
108
|
+
errors: cap_errors(all_errors)
|
|
109
|
+
}
|
|
110
|
+
end
|
|
57
111
|
end
|
|
58
112
|
|
|
59
113
|
# Sync model units to the Data Models Notion database.
|
|
60
114
|
#
|
|
61
|
-
#
|
|
115
|
+
# Pages are titled with the table name. When several models share one
|
|
116
|
+
# table (STI hierarchies, custom +self.table_name+ overlaps) a bare
|
|
117
|
+
# table-name title makes each model overwrite the others' pages (#149),
|
|
118
|
+
# so those titles are qualified as "<table> (<ModelName>)". Models with
|
|
119
|
+
# a unique table keep the bare title — no churn for the common case.
|
|
120
|
+
#
|
|
121
|
+
# Manifest entries whose title vanished from the current model set are
|
|
122
|
+
# pruned afterwards; the Notion pages themselves are left alone (no
|
|
123
|
+
# deletion path exists, and none is invented here).
|
|
124
|
+
#
|
|
125
|
+
# @return [Hash] { synced: Integer, skipped: Integer, errors: Array<String> }
|
|
62
126
|
def sync_data_models
|
|
63
127
|
database_id = @database_ids[:data_models]
|
|
64
128
|
return empty_stats unless database_id
|
|
65
129
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
130
|
+
begin
|
|
131
|
+
migration_dates = load_migration_dates
|
|
132
|
+
shared_tables = shared_table_names
|
|
133
|
+
stats = sync_units('model', database_id, 'Table Name', SCOPE_DATA_MODELS) do |unit_data|
|
|
134
|
+
properties = Mappers::ModelMapper.new.map(unit_data)
|
|
135
|
+
# Enrichment reads the bare table name from the title — run it
|
|
136
|
+
# before any qualification rewrites the title.
|
|
137
|
+
enrich_with_migration_date(properties, migration_dates)
|
|
138
|
+
legacy_title = qualify_shared_table_title(properties, unit_data, shared_tables)
|
|
139
|
+
[properties, legacy_title]
|
|
140
|
+
end
|
|
141
|
+
@manifest.prune(SCOPE_DATA_MODELS, stats.delete(:current_keys))
|
|
142
|
+
stats
|
|
143
|
+
ensure
|
|
144
|
+
save_manifest
|
|
71
145
|
end
|
|
72
146
|
end
|
|
73
147
|
|
|
74
148
|
# Sync column data to the Columns Notion database.
|
|
75
149
|
#
|
|
76
|
-
#
|
|
150
|
+
# Columns are synced **per physical table**, not per model. Column pages
|
|
151
|
+
# are titled "<table>.<column>", so two models sharing a table (STI, a
|
|
152
|
+
# shared +self.table_name=+) produced one page from two different
|
|
153
|
+
# payloads and PATCHed it back and forth on every run, forever (EXP-1).
|
|
154
|
+
# Grouping first means one page per physical column per run, with the
|
|
155
|
+
# Table relation listing every owning model and the validations unioned.
|
|
156
|
+
#
|
|
157
|
+
# Manifest entries whose qualified title vanished from the current
|
|
158
|
+
# column set are pruned afterwards; the Notion pages themselves are
|
|
159
|
+
# left alone (no deletion path exists, and none is invented here).
|
|
160
|
+
#
|
|
161
|
+
# @return [Hash] { synced: Integer, skipped: Integer, errors: Array<String> }
|
|
77
162
|
def sync_columns
|
|
78
163
|
database_id = @database_ids[:columns]
|
|
79
164
|
return empty_stats unless database_id
|
|
80
165
|
|
|
81
|
-
|
|
82
|
-
|
|
166
|
+
begin
|
|
167
|
+
totals = { synced: 0, skipped: 0, errors: [] }
|
|
168
|
+
current_keys = []
|
|
83
169
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
170
|
+
column_groups.each do |group|
|
|
171
|
+
result = sync_table_columns(group, database_id, current_keys)
|
|
172
|
+
totals[:synced] += result[:synced]
|
|
173
|
+
totals[:skipped] += result[:skipped]
|
|
174
|
+
totals[:errors].concat(result[:errors])
|
|
175
|
+
end
|
|
89
176
|
|
|
90
|
-
|
|
177
|
+
@manifest.prune(SCOPE_COLUMNS, current_keys)
|
|
178
|
+
totals
|
|
179
|
+
ensure
|
|
180
|
+
save_manifest
|
|
181
|
+
end
|
|
91
182
|
end
|
|
92
183
|
|
|
93
|
-
MAX_ERRORS = 100
|
|
94
|
-
|
|
95
184
|
private
|
|
96
185
|
|
|
97
186
|
# Sync all units of a type, yielding each for property mapping.
|
|
@@ -99,28 +188,39 @@ module Woods
|
|
|
99
188
|
# @param type [String] Unit type to list
|
|
100
189
|
# @param database_id [String] Notion database UUID
|
|
101
190
|
# @param title_property [String] Name of the title property
|
|
102
|
-
# @
|
|
103
|
-
# @
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
191
|
+
# @param scope [String] Manifest scope for this database
|
|
192
|
+
# @yield [Hash] Unit data hash; expects back +[properties, legacy_title]+
|
|
193
|
+
# where +legacy_title+ (String, nil) is a pre-qualification title an
|
|
194
|
+
# already-synced page may still carry (see {#adopt_legacy_page})
|
|
195
|
+
# @return [Hash] { synced:, skipped:, errors:, current_keys: }
|
|
196
|
+
def sync_units(type, database_id, title_property, scope)
|
|
197
|
+
stats = { synced: 0, skipped: 0, errors: [], current_keys: [] }
|
|
107
198
|
|
|
108
199
|
@reader.list_units(type: type).each do |entry|
|
|
109
200
|
unit_data = @reader.find_unit(entry['identifier'])
|
|
110
201
|
next unless unit_data
|
|
111
202
|
|
|
112
203
|
begin
|
|
113
|
-
properties = yield(unit_data)
|
|
204
|
+
properties, legacy_title = yield(unit_data)
|
|
114
205
|
title_value = extract_title_text(properties[title_property])
|
|
115
|
-
|
|
206
|
+
stats[:current_keys] << title_value
|
|
207
|
+
legacy = legacy_title ? { title: legacy_title } : nil
|
|
208
|
+
status, page_id = sync_page(scope: scope, database_id: database_id, title_value: title_value,
|
|
209
|
+
properties: properties, legacy: legacy)
|
|
116
210
|
@page_id_cache[entry['identifier']] = page_id
|
|
117
|
-
|
|
211
|
+
stats[status] += 1
|
|
212
|
+
rescue AuthenticationError
|
|
213
|
+
# Not a per-unit failure: the token is wrong or the integration is
|
|
214
|
+
# not shared with this database, so every remaining call in the run
|
|
215
|
+
# would fail too — at Notion's 3 req/sec. Abort instead of spending
|
|
216
|
+
# the whole sync proving it.
|
|
217
|
+
raise
|
|
118
218
|
rescue StandardError => e
|
|
119
|
-
errors << "#{entry['identifier']}: #{e.message}"
|
|
219
|
+
stats[:errors] << "#{entry['identifier']}: #{e.message}"
|
|
120
220
|
end
|
|
121
221
|
end
|
|
122
222
|
|
|
123
|
-
|
|
223
|
+
stats
|
|
124
224
|
end
|
|
125
225
|
|
|
126
226
|
# Iterate over loaded model units.
|
|
@@ -135,27 +235,189 @@ module Woods
|
|
|
135
235
|
end
|
|
136
236
|
end
|
|
137
237
|
|
|
138
|
-
#
|
|
238
|
+
# Run a multi-read export body against one index generation.
|
|
239
|
+
#
|
|
240
|
+
# Every public IndexReader accessor self-refreshes when the published
|
|
241
|
+
# generation moves, and the reader assigns pinning responsibility to
|
|
242
|
+
# direct callers. Unpinned, a concurrent extraction mid-sync means the
|
|
243
|
+
# model list, the unit bodies and the page-id cache can straddle two
|
|
244
|
+
# generations (EXP-5) — a model present only in the newer one gets a nil
|
|
245
|
+
# parent page and a run of avoidable churn.
|
|
246
|
+
#
|
|
247
|
+
# Guarded by +respond_to?+: injected readers (specs, embedders) need not
|
|
248
|
+
# implement pinning.
|
|
249
|
+
#
|
|
250
|
+
# @yield the export body
|
|
251
|
+
# @return [Object] the block's value
|
|
252
|
+
def with_pinned_index(&block)
|
|
253
|
+
return yield unless @reader.respond_to?(:with_pinned_generation)
|
|
254
|
+
|
|
255
|
+
@reader.with_pinned_generation(&block)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# A columns-only configuration used to make {#sync_all} return all-zero
|
|
259
|
+
# stats in silence, which is indistinguishable from breakage (EXP-11).
|
|
260
|
+
# The sync runs; only the Table relation is unavailable, because its
|
|
261
|
+
# target pages live in the Data Models database nobody configured.
|
|
139
262
|
#
|
|
140
|
-
# @return [
|
|
141
|
-
def
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
263
|
+
# @return [void]
|
|
264
|
+
def warn_columns_without_data_models
|
|
265
|
+
warn 'woods: notion sync has no data_models database configured — ' \
|
|
266
|
+
'syncing columns without a Table relation'
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# One entry per physical table: every model that owns it, the union of
|
|
270
|
+
# their columns and validations, and the Data Models pages the Table
|
|
271
|
+
# relation should list.
|
|
272
|
+
#
|
|
273
|
+
# Parent page ids come from the page-id cache, which the manifest skip
|
|
274
|
+
# path populates too — so on a warm run the Table relation is
|
|
275
|
+
# byte-identical to the previous run and unchanged columns hash stable.
|
|
276
|
+
# Owners and page ids are sorted so the payload does not depend on the
|
|
277
|
+
# order the index happens to list models in.
|
|
278
|
+
#
|
|
279
|
+
# @return [Array<Hash>] { table_name:, owners:, columns:, validations:,
|
|
280
|
+
# parent_page_ids: }
|
|
281
|
+
def column_groups
|
|
282
|
+
groups = {}
|
|
283
|
+
each_model_unit { |entry, unit_data| absorb_into_column_group(groups, entry, unit_data) }
|
|
284
|
+
groups.each_value { |group| finalize_column_group(group) }
|
|
285
|
+
groups.values
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Fold one model's columns and validations into its table's group.
|
|
289
|
+
#
|
|
290
|
+
# @return [void]
|
|
291
|
+
def absorb_into_column_group(groups, entry, unit_data)
|
|
292
|
+
table_name = Mappers::ModelMapper.table_name_for(unit_data)
|
|
293
|
+
group = (groups[table_name] ||= { table_name: table_name, owners: [], columns: {}, validations: [] })
|
|
294
|
+
group[:owners] << entry['identifier']
|
|
295
|
+
(unit_data.dig('metadata', 'columns') || []).each { |column| group[:columns][column['name']] ||= column }
|
|
296
|
+
group[:validations].concat(unit_data.dig('metadata', 'validations') || [])
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# Sort and dedupe a group so its payload does not depend on the order
|
|
300
|
+
# the index happens to list models in.
|
|
301
|
+
#
|
|
302
|
+
# @return [void]
|
|
303
|
+
def finalize_column_group(group)
|
|
304
|
+
group[:owners] = group[:owners].uniq.sort
|
|
305
|
+
group[:validations] = group[:validations].uniq
|
|
306
|
+
group[:parent_page_ids] = group[:owners].filter_map { |owner| @page_id_cache[owner] }.uniq
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# Sync the columns of one physical table.
|
|
310
|
+
#
|
|
311
|
+
# Column pages are titled "<table>.<column>" (#149). Every model shares
|
|
312
|
+
# id/created_at/updated_at, and upserting by the bare column name found
|
|
313
|
+
# the previous model's page by title equality across the whole Columns
|
|
314
|
+
# database — churning it down to one page per distinct column name.
|
|
315
|
+
# The table qualifier comes from {Mappers::ModelMapper.table_name_for},
|
|
316
|
+
# the same value the Table relation's target page is titled with.
|
|
317
|
+
#
|
|
318
|
+
# @param group [Hash] One {#column_groups} entry
|
|
319
|
+
# @param current_keys [Array<String>] Sink for this run's column titles
|
|
320
|
+
# @return [Hash] { synced: Integer, skipped: Integer, errors: Array<String> }
|
|
321
|
+
def sync_table_columns(group, database_id, current_keys)
|
|
145
322
|
mapper = Mappers::ColumnMapper.new
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
columns.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
synced += 1
|
|
323
|
+
stats = { synced: 0, skipped: 0, errors: [] }
|
|
324
|
+
owners = group[:owners].join(', ')
|
|
325
|
+
|
|
326
|
+
group[:columns].each_value do |column|
|
|
327
|
+
stats[sync_one_column(mapper, group, column, database_id, current_keys)] += 1
|
|
328
|
+
rescue AuthenticationError
|
|
329
|
+
raise # see sync_units — an auth failure dooms the whole run
|
|
154
330
|
rescue StandardError => e
|
|
155
|
-
errors << "#{
|
|
331
|
+
stats[:errors] << "#{owners}.#{column['name']}: #{e.message}"
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
stats
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# Sync one physical column's page.
|
|
338
|
+
#
|
|
339
|
+
# @return [Symbol] :synced or :skipped
|
|
340
|
+
def sync_one_column(mapper, group, column, database_id, current_keys)
|
|
341
|
+
properties = mapper.map(column, model_identifier: group[:owners].first, table_name: group[:table_name],
|
|
342
|
+
validations: group[:validations],
|
|
343
|
+
parent_page_ids: group[:parent_page_ids])
|
|
344
|
+
title_value = extract_title_text(properties['Column Name'])
|
|
345
|
+
current_keys << title_value
|
|
346
|
+
status, = sync_page(scope: SCOPE_COLUMNS, database_id: database_id, title_value: title_value,
|
|
347
|
+
properties: properties,
|
|
348
|
+
legacy: column_legacy_descriptor(column, group[:parent_page_ids].first))
|
|
349
|
+
status
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
# Sync one page through the manifest. Three tiers, cheapest first:
|
|
353
|
+
# manifest hit with an unchanged hash — skip, zero API calls; hit with
|
|
354
|
+
# a changed hash — PATCH the cached page_id directly (one call, with
|
|
355
|
+
# gone-page self-heal); miss — the full #149 path ({#upsert_page}:
|
|
356
|
+
# find by qualified title, legacy adoption, create/update). Every
|
|
357
|
+
# non-skip outcome records page_id + content hash for the next run.
|
|
358
|
+
# +force_full+ bypasses the manifest reads entirely (tier three for
|
|
359
|
+
# everything) but still records.
|
|
360
|
+
#
|
|
361
|
+
# @param scope [String] Manifest scope
|
|
362
|
+
# @param database_id [String] Notion database UUID
|
|
363
|
+
# @param title_value [String] Qualified title — the manifest key
|
|
364
|
+
# @param properties [Hash] Full property payload to write
|
|
365
|
+
# @param legacy [Hash, nil] Legacy descriptor for {#upsert_page}
|
|
366
|
+
# @return [Array(Symbol, String)] [:skipped or :synced, page id]
|
|
367
|
+
def sync_page(scope:, database_id:, title_value:, properties:, legacy: nil)
|
|
368
|
+
content_hash = SyncManifest.content_hash(properties)
|
|
369
|
+
|
|
370
|
+
unless @force_full
|
|
371
|
+
if @manifest.unchanged?(scope, title_value, content_hash)
|
|
372
|
+
return [:skipped, @manifest.page_id_for(scope, title_value)]
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
cached_id = @manifest.page_id_for(scope, title_value)
|
|
376
|
+
if cached_id && (page_id = update_cached_page(scope, title_value, cached_id, properties))
|
|
377
|
+
@manifest.record(scope: scope, key: title_value, hash: content_hash, page_id: page_id)
|
|
378
|
+
return [:synced, page_id]
|
|
379
|
+
end
|
|
156
380
|
end
|
|
157
381
|
|
|
158
|
-
|
|
382
|
+
page_id = upsert_page(database_id: database_id, title_value: title_value,
|
|
383
|
+
properties: properties, legacy: legacy)
|
|
384
|
+
@manifest.record(scope: scope, key: title_value, hash: content_hash, page_id: page_id)
|
|
385
|
+
[:synced, page_id]
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
# PATCH a page by its manifest-cached id — the one-call path for
|
|
389
|
+
# changed content. When Notion reports the page gone ({STALE_PAGE_ERROR}:
|
|
390
|
+
# deleted or archived behind the cache), warn once, drop the manifest
|
|
391
|
+
# entry, and return nil so the caller falls through to the create path
|
|
392
|
+
# (self-heal). Any other API failure propagates to the per-unit rescue
|
|
393
|
+
# so it lands in the error list with the entry retained for a retry.
|
|
394
|
+
#
|
|
395
|
+
# @return [String, nil] the page id on success, nil when the cached page is gone
|
|
396
|
+
def update_cached_page(scope, key, page_id, properties)
|
|
397
|
+
@client.update_page(page_id: page_id, properties: properties)
|
|
398
|
+
page_id
|
|
399
|
+
rescue Woods::Error => e
|
|
400
|
+
raise unless e.message.match?(STALE_PAGE_ERROR)
|
|
401
|
+
|
|
402
|
+
warn "woods: notion sync cached page #{page_id} for #{key.inspect} is gone (#{e.message}) — recreating"
|
|
403
|
+
@manifest.forget(scope, key)
|
|
404
|
+
nil
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
# Describe the pre-#149 page a column may still be stored under: the
|
|
408
|
+
# bare column name, verified against this model's Table relation when
|
|
409
|
+
# the parent Data Models page is known.
|
|
410
|
+
#
|
|
411
|
+
# @param column [Hash]
|
|
412
|
+
# @param parent_page_id [String, nil]
|
|
413
|
+
# @return [Hash] Legacy descriptor for {#upsert_page}
|
|
414
|
+
def column_legacy_descriptor(column, parent_page_id)
|
|
415
|
+
legacy = { title: column['name'].to_s }
|
|
416
|
+
if parent_page_id
|
|
417
|
+
legacy[:relation_property] = 'Table'
|
|
418
|
+
legacy[:relation_page_id] = parent_page_id
|
|
419
|
+
end
|
|
420
|
+
legacy
|
|
159
421
|
end
|
|
160
422
|
|
|
161
423
|
# Enrich model properties with migration date if available.
|
|
@@ -182,9 +444,22 @@ module Woods
|
|
|
182
444
|
|
|
183
445
|
# Upsert a Notion page: find by title, update if exists, create if not.
|
|
184
446
|
#
|
|
447
|
+
# When the qualified-title lookup misses and a +legacy+ descriptor is
|
|
448
|
+
# given, a page still carrying the pre-qualification title may be
|
|
449
|
+
# adopted instead of creating a duplicate (see {#adopt_legacy_page}).
|
|
450
|
+
# Updating the adopted page rewrites its title to the qualified form,
|
|
451
|
+
# so the legacy lookup is a one-time migration cost per page.
|
|
452
|
+
#
|
|
453
|
+
# @param database_id [String] Notion database UUID
|
|
454
|
+
# @param title_value [String] Title to find the page by
|
|
455
|
+
# @param properties [Hash] Full property payload to write
|
|
456
|
+
# @param legacy [Hash, nil] { title:, relation_property:, relation_page_id: }
|
|
185
457
|
# @return [String] Notion page ID
|
|
186
|
-
def upsert_page(database_id:, title_value:, properties:)
|
|
458
|
+
def upsert_page(database_id:, title_value:, properties:, legacy: nil)
|
|
187
459
|
existing = @client.find_page_by_title(database_id: database_id, title: title_value)
|
|
460
|
+
if existing.nil? && legacy && legacy[:title] != title_value
|
|
461
|
+
existing = adopt_legacy_page(database_id: database_id, legacy: legacy, new_title: title_value)
|
|
462
|
+
end
|
|
188
463
|
|
|
189
464
|
if existing
|
|
190
465
|
@client.update_page(page_id: existing['id'], properties: properties)
|
|
@@ -195,9 +470,129 @@ module Woods
|
|
|
195
470
|
end
|
|
196
471
|
end
|
|
197
472
|
|
|
473
|
+
# Locate a page still titled with the pre-#149 (unqualified) title so
|
|
474
|
+
# the sync can adopt it — update it in place, which also rewrites its
|
|
475
|
+
# title to the qualified form — rather than strand it next to a fresh
|
|
476
|
+
# duplicate forever.
|
|
477
|
+
#
|
|
478
|
+
# Adoption is deliberately conservative:
|
|
479
|
+
# - With a relation qualifier (Columns), the title-equality filter is
|
|
480
|
+
# AND-ed with a `relation contains` filter on the parent Data Models
|
|
481
|
+
# page, so only a legacy page actually belonging to this model can be
|
|
482
|
+
# adopted. Multiple matches are same-parent duplicates; the first is
|
|
483
|
+
# adopted with a warning.
|
|
484
|
+
# - Without one (Data Models), the page is adopted only when exactly one
|
|
485
|
+
# match exists; an ambiguous set is left alone with a warning and a
|
|
486
|
+
# fresh qualified page is created instead.
|
|
487
|
+
#
|
|
488
|
+
# @param database_id [String] Notion database UUID
|
|
489
|
+
# @param legacy [Hash] { title:, relation_property:, relation_page_id: }
|
|
490
|
+
# @param new_title [String] Qualified title the adopted page will get
|
|
491
|
+
# @return [Hash, nil] Adoptable page, or nil to create fresh
|
|
492
|
+
def adopt_legacy_page(database_id:, legacy:, new_title:)
|
|
493
|
+
results = query_legacy_pages(database_id, legacy)
|
|
494
|
+
return nil if results.empty?
|
|
495
|
+
|
|
496
|
+
if results.size > 1
|
|
497
|
+
return warn_ambiguous_legacy(legacy, results) unless legacy[:relation_page_id]
|
|
498
|
+
|
|
499
|
+
warn "woods: notion sync found #{results.size} legacy pages titled #{legacy[:title].inspect} " \
|
|
500
|
+
"for one Table relation; adopting the first (#{results.first['id']})"
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
warn "woods: notion sync adopting legacy page #{results.first['id']} " \
|
|
504
|
+
"(#{legacy[:title].inspect} -> #{new_title.inspect})"
|
|
505
|
+
results.first
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
# @param legacy [Hash]
|
|
509
|
+
# @param results [Array<Hash>]
|
|
510
|
+
# @return [nil]
|
|
511
|
+
def warn_ambiguous_legacy(legacy, results)
|
|
512
|
+
warn "woods: notion sync found #{results.size} pages sharing the legacy title " \
|
|
513
|
+
"#{legacy[:title].inspect} and cannot tell them apart; creating a qualified page " \
|
|
514
|
+
'instead — clean up the legacy pages manually'
|
|
515
|
+
nil
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# Query pages carrying the legacy title, narrowed by the parent relation
|
|
519
|
+
# when the descriptor names one. This is the only extra API call the
|
|
520
|
+
# qualified-title scheme introduces, and it runs solely on the
|
|
521
|
+
# qualified-lookup miss path.
|
|
522
|
+
#
|
|
523
|
+
# @param database_id [String]
|
|
524
|
+
# @param legacy [Hash]
|
|
525
|
+
# @return [Array<Hash>] Matching pages
|
|
526
|
+
def query_legacy_pages(database_id, legacy)
|
|
527
|
+
filter = { property: 'title', title: { equals: legacy[:title] } }
|
|
528
|
+
if legacy[:relation_property] && legacy[:relation_page_id]
|
|
529
|
+
filter = { and: [filter, { property: legacy[:relation_property],
|
|
530
|
+
relation: { contains: legacy[:relation_page_id] } }] }
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
response = @client.query_database(database_id: database_id, filter: filter)
|
|
534
|
+
response['results'] || []
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
# Table names claimed by more than one model unit in the index — STI
|
|
538
|
+
# hierarchies, or models sharing a table via +self.table_name=+.
|
|
539
|
+
#
|
|
540
|
+
# @return [Array<String>]
|
|
541
|
+
def shared_table_names
|
|
542
|
+
counts = Hash.new(0)
|
|
543
|
+
each_model_unit do |_entry, unit_data|
|
|
544
|
+
counts[Mappers::ModelMapper.table_name_for(unit_data)] += 1
|
|
545
|
+
end
|
|
546
|
+
counts.select { |_table, count| count > 1 }.keys
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# Rewrite the Data Models title to "<table> (<ModelName>)" for a model
|
|
550
|
+
# whose table name is shared with another model, so the two stop
|
|
551
|
+
# overwriting each other's pages (#149).
|
|
552
|
+
#
|
|
553
|
+
# @param properties [Hash] Mapped Notion properties (mutated)
|
|
554
|
+
# @param unit_data [Hash]
|
|
555
|
+
# @param shared_tables [Array<String>]
|
|
556
|
+
# @return [String, nil] The bare table name as the legacy title when
|
|
557
|
+
# qualification was applied, nil otherwise
|
|
558
|
+
def qualify_shared_table_title(properties, unit_data, shared_tables)
|
|
559
|
+
table_name = Mappers::ModelMapper.table_name_for(unit_data)
|
|
560
|
+
return nil unless shared_tables.include?(table_name)
|
|
561
|
+
|
|
562
|
+
qualified = "#{table_name} (#{unit_data['identifier']})"
|
|
563
|
+
properties['Table Name'] = { title: [{ text: { content: qualified } }] }
|
|
564
|
+
table_name
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
# Persist the manifest, downgrading failures to a warning: a lost
|
|
568
|
+
# manifest only costs a full re-check next run, which must not turn an
|
|
569
|
+
# otherwise-successful sync into a crash (called from ensure blocks,
|
|
570
|
+
# where a raise would also mask any in-flight exception).
|
|
571
|
+
#
|
|
572
|
+
# @return [void]
|
|
573
|
+
def save_manifest
|
|
574
|
+
@manifest.save
|
|
575
|
+
rescue StandardError => e
|
|
576
|
+
warn "woods: notion sync manifest not persisted (#{e.message}) — next run re-checks every page"
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
# @param index_dir [String]
|
|
580
|
+
# @return [SyncManifest]
|
|
581
|
+
def build_manifest(index_dir)
|
|
582
|
+
SyncManifest.new(path: File.join(index_dir, MANIFEST_FILENAME), database_ids: @database_ids)
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
# Truthy set mirrors the rake tasks' env_flag convention, so
|
|
586
|
+
# WOODS_NOTION_FORCE=false / =0 disables rather than silently enabling.
|
|
587
|
+
#
|
|
588
|
+
# @return [Boolean]
|
|
589
|
+
def env_force?
|
|
590
|
+
%w[1 true yes].include?(ENV.fetch(FORCE_ENV_VAR, '').strip.downcase)
|
|
591
|
+
end
|
|
592
|
+
|
|
198
593
|
# @return [Hash]
|
|
199
594
|
def empty_stats
|
|
200
|
-
{ synced: 0, errors: [] }
|
|
595
|
+
{ synced: 0, skipped: 0, errors: [] }
|
|
201
596
|
end
|
|
202
597
|
|
|
203
598
|
# Cap errors to prevent unbounded memory growth.
|