@redocly/framework 1.0.0-next.5 → 1.0.0-next.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundler/build/output-metadata.js +1 -1
- package/dist/bundler/build/run.js +1 -1
- package/dist/bundler/build/theme-global-style-compiler.js +1 -0
- package/dist/bundler/build/theme-global-stylesheets.js +1 -0
- package/dist/bundler/build/vite-config.js +2 -2
- package/dist/bundler/dev/materialization-sweeper.js +1 -1
- package/dist/bundler/dev/run.js +1 -1
- package/dist/bundler/dev/runtime-app.js +1 -1
- package/dist/bundler/node-bundle.js +1 -0
- package/dist/bundler/runtime-files.js +1 -1
- package/dist/bundler/user-markdoc-schema.js +1 -1
- package/dist/bundler/vite-config.base.js +12 -12
- package/dist/cli/commands/ingest.js +1 -1
- package/dist/cli/commands/translate.js +1 -1
- package/dist/cli/validate-react-pages.js +1 -1
- package/dist/client/ApiAccess.js +3 -0
- package/dist/client/NavigationProgress.js +1 -1
- package/dist/client/ProductsProvider.js +1 -1
- package/dist/client/RouteErrorBoundary.js +1 -1
- package/dist/client/ThemeGlobalStyle.js +1 -0
- package/dist/client/UnversionedVersionButton.js +1 -1
- package/dist/client/app.js +3 -3
- package/dist/client/client-entry.js +1 -1
- package/dist/client/dev-problems/ProblemsBar.js +6 -6
- package/dist/client/dev-problems/ProblemsModal.js +15 -15
- package/dist/client/l10n/use-l10n-config.js +1 -1
- package/dist/client/l10n/use-l10n-href.js +1 -1
- package/dist/client/l10n/use-translate.js +1 -1
- package/dist/client/products.js +1 -1
- package/dist/client/review/ReviewBar.js +53 -53
- package/dist/client/sidebar/PoweredByRedocly.js +2 -2
- package/dist/client/ssr.js +1 -1
- package/dist/client/telemetry/report-preload-failure.js +1 -0
- package/dist/client/templates/dev-login.js +4 -4
- package/dist/client/templates/unauthorized.js +5 -5
- package/dist/client/theme-global-css.js +1 -0
- package/dist/client/use-banner.js +1 -1
- package/dist/client/use-search.js +1 -1
- package/dist/content-layer/allowed-packages.js +1 -1
- package/dist/content-layer/config-schema.d.ts +5 -1
- package/dist/content-layer/config-schema.js +1 -1
- package/dist/content-layer/config.d.ts +6 -2
- package/dist/content-layer/config.js +1 -1
- package/dist/content-layer/database/adapters/node-sqlite-client.js +1 -1
- package/dist/content-layer/database/connection.js +1 -1
- package/dist/content-layer/database/entity-store.d.ts +9 -1
- package/dist/content-layer/database/entity-store.js +1 -1
- package/dist/content-layer/database/origins.d.ts +2 -4
- package/dist/content-layer/database/origins.js +1 -1
- package/dist/content-layer/ingest/extract-all-phase.js +1 -1
- package/dist/content-layer/ingest/finalize-phase.js +1 -1
- package/dist/content-layer/ingest/hook-inputs.js +1 -1
- package/dist/content-layer/ingest/l10n-warnings.js +1 -1
- package/dist/content-layer/ingest/rbac-teams.js +1 -1
- package/dist/content-layer/ingest/run.js +1 -1
- package/dist/content-layer/ingest/scan.d.ts +1 -1
- package/dist/content-layer/ingest/scan.js +1 -1
- package/dist/content-layer/ingest/search-sync.js +1 -1
- package/dist/content-layer/ingest/slugger/resolve-final-slugs.js +1 -1
- package/dist/content-layer/ingest/sync.js +1 -1
- package/dist/content-layer/loaders/context.js +1 -1
- package/dist/content-layer/markdown/client/OpenapiTagComponents.js +2 -2
- package/dist/content-layer/markdown/client/dynamic-components.js +1 -1
- package/dist/content-layer/markdown/client/markdoc-components.js +2 -2
- package/dist/content-layer/markdown/constants/tag-sets.js +1 -1
- package/dist/content-layer/markdown/nodes/fence.js +1 -1
- package/dist/content-layer/markdown/nodes/table.js +1 -1
- package/dist/content-layer/markdown/resolve/audience-tag.d.ts +1 -1
- package/dist/content-layer/markdown/resolve/audience-tag.js +1 -1
- package/dist/content-layer/markdown/resolve/markdoc-resolvers/operation-samples.d.ts +1 -1
- package/dist/content-layer/markdown/resolve/markdoc-resolvers/operation-samples.js +1 -1
- package/dist/content-layer/markdown/resolve/markdoc-resolvers/partial.js +1 -1
- package/dist/content-layer/markdown/search/to-markdown.js +2 -2
- package/dist/content-layer/markdown/search/to-sections.js +1 -1
- package/dist/content-layer/markdown/server/render.js +1 -1
- package/dist/content-layer/markdown/service.js +1 -1
- package/dist/content-layer/markdown/tags/html.d.ts +4 -1
- package/dist/content-layer/markdown/tags/html.js +1 -1
- package/dist/content-layer/markdown/types/service.d.ts +2 -0
- package/dist/content-layer/materialization/entity.js +1 -1
- package/dist/content-layer/materialization/materializer.js +1 -0
- package/dist/content-layer/materialization/static-files.js +1 -1
- package/dist/content-layer/plan-gates.d.ts +3 -2
- package/dist/content-layer/plan-gates.js +1 -1
- package/dist/content-layer/plugins/index.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/client/template.js +4 -4
- package/dist/content-layer/plugins/internal/api-docs/index.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/server/build-definition.d.ts +2 -2
- package/dist/content-layer/plugins/internal/api-docs/server/build-definition.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/server/build-sidebar.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/server/markdown-docs.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/server/options.d.ts +4 -2
- package/dist/content-layer/plugins/internal/api-docs/server/options.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/server/search-documents.js +1 -1
- package/dist/content-layer/plugins/internal/api-docs/server/spec-download.js +1 -1
- package/dist/content-layer/plugins/internal/api-functions/index.js +1 -1
- package/dist/content-layer/plugins/internal/api-functions/server/access.js +1 -1
- package/dist/content-layer/plugins/internal/api-functions/server/dispatch.js +1 -1
- package/dist/content-layer/plugins/internal/app-data/banner.js +1 -1
- package/dist/content-layer/plugins/internal/app-data/index.js +1 -1
- package/dist/content-layer/plugins/internal/app-data/products.d.ts +2 -0
- package/dist/content-layer/plugins/internal/app-data/products.js +1 -1
- package/dist/content-layer/plugins/internal/index.js +1 -1
- package/dist/content-layer/plugins/internal/l10n/index.js +1 -1
- package/dist/content-layer/plugins/internal/markdoc/client/template.js +1 -1
- package/dist/content-layer/plugins/internal/markdoc/index.js +1 -1
- package/dist/content-layer/plugins/internal/markdoc/server/shared.d.ts +1 -1
- package/dist/content-layer/plugins/internal/mcp/content-tools.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/index.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/agent-card.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/audit.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/auth.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/config-gate.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/custom-tools.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/gateway/__tests__/secret-store-fixture.js +1 -0
- package/dist/content-layer/plugins/internal/mcp/server/gateway/credentials-page/form-page.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/gateway/credentials.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/gateway/elicitation.js +7 -7
- package/dist/content-layer/plugins/internal/mcp/server/gateway/fetch/fetch-bridge.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/gateway/index.js +3 -3
- package/dist/content-layer/plugins/internal/mcp/server/gateway/secret-store.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/gateway/tools.js +3 -3
- package/dist/content-layer/plugins/internal/mcp/server/handler.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/mcp-server.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/oauth/authorize.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/oauth/discovery.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/server-card.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/skills-resources.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/tools/graphql-catalog.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/tools/openapi-catalog.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/tools/openapi-document.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/tools/openapi-refs.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/tools/openapi.js +1 -1
- package/dist/content-layer/plugins/internal/mcp/server/tools/pages-navigation.js +1 -1
- package/dist/content-layer/plugins/internal/mock/server/handler.js +1 -1
- package/dist/content-layer/plugins/internal/react-page/index.js +1 -1
- package/dist/content-layer/plugins/internal/scorecard-classic/client/Grid.js +11 -11
- package/dist/content-layer/plugins/internal/scorecard-classic/client/ScorecardViews.js +1 -1
- package/dist/content-layer/plugins/internal/scorecard-classic/client/ScorecardWidgets.js +7 -7
- package/dist/content-layer/plugins/internal/scorecard-classic/client/template.js +2 -2
- package/dist/content-layer/plugins/internal/sidebar/bundle.d.ts +1 -1
- package/dist/content-layer/plugins/internal/sidebar/bundle.js +1 -1
- package/dist/content-layer/plugins/internal/sidebar/index.d.ts +1 -1
- package/dist/content-layer/plugins/internal/sidebar/index.js +1 -1
- package/dist/content-layer/plugins/internal/skills/index.js +1 -1
- package/dist/content-layer/plugins/internal/skills/server/routes.js +1 -1
- package/dist/content-layer/plugins/project/load.js +2 -2
- package/dist/content-layer/plugins/registry.js +1 -1
- package/dist/content-layer/plugins/request-context.d.ts +5 -0
- package/dist/content-layer/plugins/types/api-entity.d.ts +2 -6
- package/dist/content-layer/plugins/types/api-entity.js +1 -1
- package/dist/content-layer/plugins/types/plugin.d.ts +8 -0
- package/dist/content-layer/plugins/types/search.d.ts +1 -1
- package/dist/content-layer/plugins/utils/route-index.d.ts +1 -1
- package/dist/content-layer/plugins/utils/route-index.js +1 -1
- package/dist/content-layer/plugins/utils/version-path.js +1 -1
- package/dist/content-layer/revisions/config.js +1 -1
- package/dist/content-layer/server/context.js +1 -1
- package/dist/content-layer/server/materialize.js +1 -1
- package/dist/content-layer/static/audience.js +1 -1
- package/dist/content-layer/static/registry.js +1 -1
- package/dist/content-layer/templates/template-set.js +1 -1
- package/dist/content-layer/versions/redirect.d.ts +11 -6
- package/dist/content-layer/versions/redirect.js +1 -1
- package/dist/plugin-api.d.ts +3 -0
- package/dist/plugin-api.js +1 -1
- package/dist/server/__tests__/rbac-fixture.js +1 -1
- package/dist/server/app.js +2 -2
- package/dist/server/ask-ai/feature-gate.js +1 -1
- package/dist/server/ask-ai/route.js +1 -1
- package/dist/server/auth/gate.js +1 -1
- package/dist/server/auth/middleware.js +1 -1
- package/dist/server/auth/request.js +1 -1
- package/dist/server/auth/routes.js +1 -1
- package/dist/server/auth/runtime.js +1 -1
- package/dist/server/content-assets/access.js +1 -1
- package/dist/server/content.js +12 -12
- package/dist/server/feedback/route.js +1 -1
- package/dist/server/lambda-entry.js +1 -1
- package/dist/server/metrics/request-metrics.js +1 -1
- package/dist/server/node-entry.js +1 -1
- package/dist/server/plugin-context.js +1 -1
- package/dist/server/plugin-routes.js +1 -1
- package/dist/server/rbac/content-asset-access.js +1 -1
- package/dist/server/replay-oauth2/route.js +1 -1
- package/dist/server/review/route.js +1 -1
- package/dist/server/runtime-config.js +1 -1
- package/dist/server/search/route.js +1 -1
- package/dist/server/static-files/serve.js +1 -1
- package/dist/server/subprocess/runner.js +1 -1
- package/dist/server/telemetry/route.js +1 -1
- package/dist/server/template-ids.js +1 -1
- package/dist/shared/constants/api-spec-kinds.d.ts +6 -0
- package/dist/shared/constants/api-spec-kinds.js +1 -0
- package/dist/shared/constants/l10n/langs/ar.js +1 -1
- package/dist/shared/constants/l10n/langs/de.js +1 -1
- package/dist/shared/constants/l10n/langs/en.js +1 -1
- package/dist/shared/constants/l10n/langs/es.js +1 -1
- package/dist/shared/constants/l10n/langs/fr.js +1 -1
- package/dist/shared/constants/l10n/langs/hi.js +1 -1
- package/dist/shared/constants/l10n/langs/it.js +1 -1
- package/dist/shared/constants/l10n/langs/ja.js +1 -1
- package/dist/shared/constants/l10n/langs/ko.js +1 -1
- package/dist/shared/constants/l10n/langs/pl.js +1 -1
- package/dist/shared/constants/l10n/langs/pt-br.js +1 -1
- package/dist/shared/constants/l10n/langs/pt.js +1 -1
- package/dist/shared/constants/l10n/langs/ru.js +1 -1
- package/dist/shared/constants/l10n/langs/uk.js +1 -1
- package/dist/shared/constants/l10n/langs/zh.js +1 -1
- package/dist/{server → shared}/constants/system-routes.d.ts +1 -1
- package/dist/shared/constants/system-routes.js +1 -0
- package/dist/shared/plan-gates.d.ts +6 -1
- package/dist/shared/plan-gates.js +1 -1
- package/dist/shared/products.js +1 -1
- package/dist/{server → shared}/project-env.d.ts +1 -1
- package/dist/{server → shared}/rbac/api-docs.d.ts +1 -1
- package/dist/{server → shared}/rbac/engine.d.ts +1 -1
- package/dist/shared/rbac/engine.js +0 -0
- package/dist/shared/rbac/filter.js +1 -0
- package/dist/{server → shared}/rbac/l10n-rules.d.ts +1 -1
- package/dist/shared/rbac/l10n-rules.js +1 -0
- package/dist/shared/rbac/route-scope.js +1 -0
- package/dist/{server → shared}/rbac/team-templates.d.ts +6 -2
- package/dist/shared/rbac/team-templates.js +1 -0
- package/dist/{server → shared}/rbac/teams.d.ts +1 -1
- package/dist/shared/rbac/types.js +1 -0
- package/dist/shared/types/config.d.ts +2 -0
- package/dist/shared/utils/l10n.js +1 -1
- package/package.json +8 -8
- package/dist/content-layer/server/config.js +0 -1
- package/dist/server/constants/system-routes.js +0 -1
- package/dist/server/rbac/engine.js +0 -0
- package/dist/server/rbac/filter.js +0 -1
- package/dist/server/rbac/l10n-rules.js +0 -1
- package/dist/server/rbac/route-scope.js +0 -1
- package/dist/server/rbac/team-templates.js +0 -1
- package/dist/server/rbac/types.js +0 -1
- /package/dist/{server → shared}/project-env.js +0 -0
- /package/dist/{server → shared}/rbac/api-docs.js +0 -0
- /package/dist/{server → shared}/rbac/entity-scope.js +0 -0
- /package/dist/{server → shared}/rbac/filter.d.ts +0 -0
- /package/dist/{server → shared}/rbac/route-scope.d.ts +0 -0
- /package/dist/{server → shared}/rbac/teams.js +0 -0
- /package/dist/{server → shared}/rbac/types.d.ts +0 -0
- /package/dist/{content-layer/markdown → shared}/utils/packed-ast.d.ts +0 -0
- /package/dist/{content-layer/markdown → shared}/utils/packed-ast.js +0 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { PlatformFs } from '../platform/types/fs.js';
|
|
2
|
-
import { type TeamDirs } from '../
|
|
2
|
+
import { type TeamDirs } from '../shared/rbac/team-templates.js';
|
|
3
3
|
import { type ProcessEnv } from '../shared/public-env.js';
|
|
4
4
|
import type { NestedConfig, RealmConfig } from '../shared/types/config.js';
|
|
5
|
+
import { type ConfigSchema } from './config-schema.js';
|
|
5
6
|
import type { ContentLayerContext } from './context.js';
|
|
6
7
|
import type { ProblemDraft } from './problems/types.js';
|
|
7
8
|
export type { RealmConfig } from '../shared/types/config.js';
|
|
@@ -37,8 +38,9 @@ type ConfigObject = Record<string, unknown>;
|
|
|
37
38
|
* `REDOCLY_ENFORCE_LOGIN` and `REDOCLY_ENFORCE_RESIDENCY` enforce applied. `siteConfig`
|
|
38
39
|
* keeps the `{{ process.env.X }}` references, judged by their values in the environment;
|
|
39
40
|
* `config` has them replaced. A reference reads the view a host gives it (`readProjectEnv`).
|
|
41
|
+
* `pluginSchemas` checks the root key of each plugin id it names.
|
|
40
42
|
*/
|
|
41
|
-
export declare function loadRedoclyConfig(contentDir: string, fs: PlatformFs): Promise<LoadedRedoclyConfig>;
|
|
43
|
+
export declare function loadRedoclyConfig(contentDir: string, fs: PlatformFs, pluginSchemas?: Readonly<Record<string, ConfigSchema>>): Promise<LoadedRedoclyConfig>;
|
|
42
44
|
/**
|
|
43
45
|
* Whether the effective config differs from the one whose digest the last ingest stored;
|
|
44
46
|
* `null` (no ingest yet, or a row an older ingest wrote) counts as changed. The effective config
|
|
@@ -80,11 +82,13 @@ export declare function deepMergeConfig(base: ConfigObject, override: ConfigObje
|
|
|
80
82
|
* `access.rbac.content` and `redirects` cover every content locale, `access.requiresLogin`
|
|
81
83
|
* follows the rbac rules when `access.rbac` is set, and the `PUBLIC_*` environment is
|
|
82
84
|
* attached. `report` receives one problem per removed, ignored, or deprecated setting.
|
|
85
|
+
* `pluginSchemas` checks the root key of each plugin id it names.
|
|
83
86
|
*/
|
|
84
87
|
export declare function normalizeRedoclyConfig(config: Record<string, unknown>, options?: {
|
|
85
88
|
dirLocales?: readonly string[];
|
|
86
89
|
teamDirs?: TeamDirs;
|
|
87
90
|
env?: ProcessEnv;
|
|
91
|
+
pluginSchemas?: Readonly<Record<string, ConfigSchema>>;
|
|
88
92
|
report?: (problem: ProblemDraft) => void;
|
|
89
93
|
}): RealmConfig;
|
|
90
94
|
export declare function createEmptyRealmConfig(): RealmConfig;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{toJsonObject as e,toJsonValue as t}from"./database/utils/json.js";import{hashContentBytes as n}from"../shared/utils/content-hash.js";import{
|
|
1
|
+
import{toJsonObject as e,toJsonValue as t}from"./database/utils/json.js";import{hashContentBytes as n}from"../shared/utils/content-hash.js";import{getProcessEnv as r,getPublicEnv as i,resolveRedoclyEnvironment as a,sanitizeBranchName as o}from"../shared/public-env.js";import{isClosedToAnonymous as s,parseRbacConfigData as c}from"../shared/rbac/engine.js";import{readProjectEnv as l}from"../shared/project-env.js";import{listL10nDirs as u,resolveLocaleContext as d}from"../shared/utils/l10n.js";import{expandConfigRbacForL10n as f}from"../shared/rbac/l10n-rules.js";import{expandConfigRbacTeamTemplates as p,listTeamDirs as m,teamTemplatePrefixes as ee}from"../shared/rbac/team-templates.js";import{resolveContentPath as h}from"../shared/utils/resolve-content-path.js";import{resolveConfigRedirects as g}from"./config-redirects.js";import{NESTED_CONFIG_SCHEMA as _,realmConfigSchema as v,sanitizeConfig as y}from"./config-schema.js";import{hashEffectiveConfig as b,resolveSiteConfig as x}from"./database/site-config.js";import{applyEnforcedAccess as S}from"./enforced-access.js";import{applyPlanGates as C}from"./plan-gates.js";import{resolve as w}from"node:path";import T from"js-yaml";import{interpolateEnvVariables as E}from"@redocly/config";const D=[`redocly.yaml`,`redocly.yml`],O=`redocly.yaml`;async function k(e,t,i={}){let a=await re(e,t),o=r();if(!a){let e=[],t=S(C($(),o,t=>e.push({code:`configuration/ignored`,message:t})),o);return{absolutePath:null,config:t,siteConfig:t,path:null,hash:null,refs:[],problems:e}}let s=await t.readBinaryFile(a.absolutePath),c=[],d=[],f=[],p=P(new TextDecoder().decode(s),a.path),h=[],g=l(o),_={contentDir:e,fs:t,env:g,refs:f,warnings:c,errors:d},v=Q(await V(await L(await F(p,{..._,deprecations:h}),``,_),_)),y={filePath:a.path},b=[],w=U(v.access)?v.access.rbac:void 0,T=K(v,{dirLocales:await u(e,t),teamDirs:await m(e,t,ee(U(w)?w:{})),env:g,pluginSchemas:i,report:e=>b.push({...e,location:y})}),E=S(C(T,o,e=>b.push({code:`configuration/ignored`,message:e,location:y})),o),{config:D,unsetVars:O}=x(E,g);for(let e of O)c.push(`The config references process.env.${e}, which is not set.`);let k=[...c.map(e=>({code:`configuration/ignored`,message:e,location:y})),...[...new Set(h)].map(e=>({code:`configuration/deprecated`,message:e,location:y})),...d.map(e=>({code:`configuration/invalid`,severity:`error`,message:e,location:y})),...b];return{...a,config:D,siteConfig:E,hash:await n(s),refs:f,problems:k}}function A(e,t){return b(e)!==t}async function j(e){if(e.context.configLoaded&&e.force!==!0)return{absolutePath:e.context.configFilePath?w(e.context.contentDir,e.context.configFilePath):null,config:e.context.config,siteConfig:e.context.siteConfig,path:e.context.configFilePath,hash:e.context.configHash,refs:e.context.configRefs,problems:[]};let t=Object.fromEntries(e.context.pluginRegistry.list().flatMap(e=>e.configSchema?[[e.id,e.configSchema]]:[])),n=await k(e.context.contentDir,e.context.platform.fs,t);return e.context.config=n.config,e.context.siteConfig=n.siteConfig,e.context.configFilePath=n.path,e.context.configHash=n.hash,e.context.configRefs=n.refs,e.context.configLoaded=!0,n}function M(e,t){let n=E(P(e,t),l(r())).value,{value:i,problems:a}=y(_,n);for(let e of a)e.location={filePath:t};return{config:i,problems:a}}function te(e){return e.endsWith(`/${O}`)}async function N(e){let t=[];for(let n of e.paths)try{let r=await e.fs.readFile(w(e.contentDir,n));t.push(...M(r,n).problems)}catch(e){let r=e instanceof Error?e.message:String(e);t.push({code:`configuration/invalid`,message:`The config cannot be read: ${r}`,location:{filePath:n}})}return t}function P(t,n){let r=T.load(t);if(r==null)return{};try{return e(r)}catch(e){throw Error(`Expected ${n} to contain a YAML object.`,{cause:e})}}async function F(e,t){let{env:n,...r}=e;if(!U(n))return e;let i=t.env.PUBLIC_REDOCLY_BRANCH_NAME,s=[...i?[`branch.${o(i)}`]:[],a(t.env.REDOCLY_ENV)],c=e=>t.deprecations.push(e),l=J(r,c);for(let e of s){let r=await I(n[e],e,t);r&&(l=H(l,J(r,c)))}return l}async function I(e,t,n){if(!U(e))return null;if(typeof e.$ref!=`string`)return e;let{$ref:r,...i}=e,a=`env.${t}`,o=R(await z(r,a,n),i,a,n);return U(o)?o:null}async function L(e,t,n){if(Array.isArray(e))return Promise.all(e.map((e,r)=>L(e,`${t}[${r}]`,n)));if(!U(e))return e;if(typeof e.$ref==`string`){let{$ref:r,...i}=e;return R(await z(r,t,n),await L(i,t,n),t,n)}let r={};for(let[i,a]of Object.entries(e))r[i]=await L(a,t===``?i:`${t}.${i}`,n);return r}function R(e,t,n,r){if(Object.keys(t).length===0)return e;if(U(e))return{...e,...t};if(e==null)return t;let i=Array.isArray(e)?`list`:`scalar`;return r.warnings.push(`${n===``?`The config`:n} has keys next to a $ref to a ${i}; only the referenced file applies.`),e}function z(e,n,r){return B(e,`${n} $ref`,r,e=>t(T.load(e)))}async function B(e,t,r,i,a=`warning`){let o=a===`error`?r.errors:r.warnings,s=h(e,D[0]);if(s===null)return o.push(`${t} "${e}" escapes the content root.`),null;try{let e=await r.fs.readBinaryFile(w(r.contentDir,s));return r.refs.push({path:s,hash:await n(e)}),i(new TextDecoder().decode(e))}catch(n){let r=n instanceof Error?n.message:String(n);return o.push(`Cannot read ${t} "${e}": ${r}`),null}}async function V(e,t){let{seo:n}=e;if(!U(n)||!U(n.llmstxt)||!U(n.llmstxt.details))return e;let{path:r,...i}=n.llmstxt.details;if(typeof r!=`string`)return e;let a=null;i.content===void 0?a=await B(E(r,t.env).value,`seo.llmstxt.details.path`,t,e=>e,`error`):t.errors.push(`seo.llmstxt.details.path and seo.llmstxt.details.content are mutually exclusive; the path is ignored.`);let o={...n.llmstxt,details:a===null?i:{...i,path:r,content:a}};return{...e,seo:{...n,llmstxt:o}}}function H(e,t){let n={...e};for(let[e,r]of Object.entries(t)){if(e===`__proto__`||e===`constructor`||e===`prototype`)continue;let t=n[e];n[e]=U(t)&&U(r)?H(t,r):r}return n}function U(e){return typeof e==`object`&&!!e&&!Array.isArray(e)}const W=[`hide`,`name`,`ignore`,`publicEndpoint`],G=[`requiresLogin`,`rbac`,`sso`,`idps`,`logoutReturnUrl`,`residency`];function K(e,t={}){let n=t.report??(()=>{}),r=t.dirLocales??[],a=J(e,e=>n({code:`configuration/deprecated`,message:e}));U(a.mcp)&&a.mcp.docs!==void 0&&n({code:`configuration/deprecated`,message:`The 'mcp.docs' property is deprecated. Use the 'mcp' properties instead.`});let o=y(v(t.pluginSchemas??{}),a,{env:t.env});for(let e of o.problems)n(e);let{redirects:s,access:c,...l}=f(p(q(o.value,t.env),t.teamDirs??{},n),r),u={...l,publicEnv:i()};c&&(u.access=ne(c,n));let m=g(s,d(l.l10n,r));return m&&(u.redirects=m),u}function q(e,t){if(!t)return e;let n={...e};for(let r of[`access`,`l10n`,`redirects`])r in e&&(n[r]=E(e[r],t).value);return n}function J(e,t){return Z(Y(Q(e),t))}function Y(e,t){let n=e;e.reunite!==void 0&&(t(`The 'reunite' property is deprecated. Use 'platform' instead.`),e.platform!==void 0&&e.platform!==null&&t(`Both 'platform' and 'reunite' are set. 'reunite' is ignored, so remove it.`),n=X(e));let{access:r}=n;return U(r)&&U(r.rbac)&&r.rbac.reunite!==void 0&&(t(`The 'access.rbac.reunite' property is deprecated. Use 'access.rbac.platform' instead.`),n={...n,access:{...r,rbac:X(r.rbac)}}),n}function X(e){let{reunite:t,...n}=e;return{...n,platform:e.platform??t}}function Z(e){let{mcp:t}=e;if(!U(t))return e;let{docs:n,...r}=t;if(!U(n))return e;let i={...n};for(let e of W){let t=n[e];delete i[e],t!==void 0&&(r[e]===void 0||r[e]===null||e===`hide`&&t===!0)&&(r[e]=t)}return{...e,mcp:{...r,docs:i}}}function Q(e){let t=G.map(t=>[t,e[t]]).filter(([,e])=>e!==void 0);if(t.length===0)return e;let n={...e},r=U(e.access)?{...e.access}:{};for(let[e,i]of t)delete n[e],(r[e]===void 0||r[e]===null)&&(r[e]=i);return{...n,access:r}}function ne(e,t){let n=e.rbac?c(e.rbac,e=>t({code:`configuration/invalid`,message:e})):null;return n?(e.requiresLogin===!0&&t({code:`configuration/ignored`,message:`access.requiresLogin is ignored because access.rbac is also configured — the two are mutually exclusive. Remove one.`}),{...e,requiresLogin:s(n)}):e}async function re(e,t){for(let n of D){let r=w(e,n);if(await t.exists(r))return{absolutePath:r,path:n}}return null}function $(){return K({})}export{O as NESTED_CONFIG_FILENAME,D as REDOCLY_CONFIG_FILENAMES,$ as createEmptyRealmConfig,H as deepMergeConfig,A as isConfigChanged,te as isNestedConfigPath,j as loadContentLayerConfig,k as loadRedoclyConfig,K as normalizeRedoclyConfig,M as parseNestedConfig,N as readNestedConfigProblems};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{createSqliteDatabaseClient as e}from"@redocly/realm-db-client";import{DatabaseSync as t}from"node:sqlite";function n(n){let i=new t(n,{timeout:5e3}),o=/* @__PURE__ */ new Map;function s(e){let t=o.get(e);if(t)return o.delete(e),o.set(e,t),t;let n=i.prepare(e);return n.setReturnArrays(!0),n.setReadBigInts(!0),o.size>=256&&o.delete(o.keys().next().value),o.set(e,n),n}return e({exec:e=>i.exec(e),run({sql:e,args:t}){let n=s(e),i=r(t);if(n.columns().length===0){let e=n.run(...i);return{columns:[],rows:[],rowsAffected:Number(e.changes),lastInsertRowid:BigInt(e.lastInsertRowid)}}let o=n.all(...i),c=n.columns();return{columns:c.map(e=>e.name),columnTypes:c.map(e=>(e.type??``).toUpperCase()),rows:o.map(e=>e.map(a))}},close(){o.clear(),i.close()}})}function r(e){if(e===void 0)return[];if(Array.isArray(e))return e.map(i);let t={};for(let[n,r]of Object.entries(e))t[n]=i(r);return[t]}function i(e){return typeof e==`boolean`?+!!e:e}function a(e){return typeof e==`bigint`&&e>=BigInt(-(2**53-1))&&e<=BigInt(2**53-1)?Number(e):e}export{n as createNodeSqliteClient};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{getRealmDir as e}from"../../shared/constants/runtime-layout.js";import{createNodePlatformFs as t}from"../../platform/node/fs.js";import{createCloudDatabaseConnection as n}from"./adapters/cloud.js";import{createLocalDatabaseConnection as r}from"./adapters/local.js";import{isRemoteDatabaseUrl as i}from"./utils/is-remote-database-url.js";import{fileURLToPath as a,pathToFileURL as o}from"node:url";import{dirname as s,join as c}from"node:path";import{sql as l}from"drizzle-orm";const u=`content-layer.db`,d=5e3,f=/* @__PURE__ */ new Map,p=/* @__PURE__ */ new Map;function m(e){let t=/^[a-z][a-z0-9+.-]*:/i.exec(e);return t?p.get(t[0].toLowerCase()):void 0}const h=a(new URL(`../../../drizzle`,import.meta.url));async function g(e){let a=e.migrationsDir??h,o=e.databaseUrl??_(e),c=e.logger,l=e.platform?.fs??t(),u,d=m(o);if(d)u=await d(o);else if(i(o))u=await n({url:o,authToken:e.authToken,instrumentation:e.instrumentation});else{let e=v(o);await l.exists(s(e))||await l.mkdir(s(e),{recursive:!0}),u=await r({path:e}),await E(u.db,c)}return e.runMigrations!==!1&&await x(u.db,{databaseUrl:o,migrationsDir:a,logger:c,operation:`run content-layer migrations`}),{db:u.db,url:o,close:async()=>{u.close()}}}function _(t){if(!t.contentDir)throw Error(`contentDir is required when databaseUrl is not provided.`);return o(c(e(t.contentDir),u)).href}function v(e){if(e.startsWith(`file:///`))return a(e);if(e.startsWith(`file:`))return e.slice(5);if(!y(e))return e;throw Error(`Unsupported content-layer database URL "${e}". Expected a file path, a file: URL, an http(s) URL, or a scheme registered with registerDatabaseUrlSchemeAdapter.`)}function y(e){return/^[a-z][a-z0-9+.-]*:/i.test(e)}async function b(e,t){let{migrate:n}=await import(`drizzle-orm/libsql/migrator`);await S(e,t),await n(e,{migrationsFolder:t})}async function x(e,t){await T({databaseUrl:t.databaseUrl,migrationsDir:t.migrationsDir,run:()=>D(()=>b(e,t.migrationsDir),{logger:t.logger,operation:t.operation})})}async function S(e,t){let{readMigrationFiles:n}=await import(`drizzle-orm/migrator`);if((await e.all(l`select 1 from sqlite_master where name = '__drizzle_migrations'`)).length===0)return;let r=new Map(n({migrationsFolder:t}).map(e=>[e.name,e.hash]));if(!(await e.all(l`select name, hash from __drizzle_migrations`)).some(e=>!r.has(e.name)||r.get(e.name)!==e.hash))return;let i=await e.all(l`select name from sqlite_master where type = 'table'
|
|
2
2
|
and name not like 'sqlite\\_%' escape '\\'
|
|
3
3
|
and name not like '\\_\\_realm\\_db\\_%' escape '\\'`);await e.run(l.raw(`PRAGMA foreign_keys = OFF`));for(let t of i)await e.run(l`drop table ${l.identifier(t.name)}`);await e.run(l.raw(`PRAGMA foreign_keys = ON`))}function C(e){if(e===void 0)return 500;if(!Number.isInteger(e)||e<=0)throw Error(`Invalid batch size: ${e}`);return e}function*w(e,t){for(let n=0;n<e.length;n+=t)yield e.slice(n,n+t)}async function T(e){let t=`${e.databaseUrl}::${e.migrationsDir}`,n=f.get(t);if(n){await n;return}let r=e.run().finally(()=>{f.delete(t)});f.set(t,r),await r}async function E(e,t){await e.run(l.raw(`PRAGMA busy_timeout = ${d}`)),await D(async()=>{await e.run(l.raw(`PRAGMA foreign_keys = ON`)),await e.run(l.raw(`PRAGMA journal_mode = WAL`)),await e.run(l.raw(`PRAGMA synchronous = NORMAL`))},{logger:t,operation:`configure local SQLite pragmas`})}async function D(e,t){let n=t.maxAttempts??4,r=t.retryDelayMs??50;for(let i=1;i<=n;i+=1)try{return await e()}catch(e){if(!O(e)||i===n)throw e;let a=r*i;t.logger?.warn(`SQLite database is busy; retrying ${t.operation} (attempt ${i}/${n}, ${a}ms backoff)`),await k(a)}throw Error(`Failed to ${t.operation}.`)}function O(e){return e instanceof Error?`code`in e&&e.code===`SQLITE_BUSY`||`errcode`in e&&e.errcode===5||e.message.includes(`SQLITE_BUSY`)||e.message.includes(`database is locked`)?!0:`cause`in e&&O(e.cause):!1}function k(e){return new Promise(t=>{setTimeout(t,e)})}export{u as CONTENT_LAYER_DB_FILENAME,d as DEFAULT_SQLITE_BUSY_TIMEOUT_MS,w as batchValues,C as normalizeBatchSize,g as openContentLayerDatabase,v as resolveLocalDatabasePath,D as runWithSqliteBusyRetry};
|
|
@@ -41,7 +41,10 @@ export type EntityStoreOptions = {
|
|
|
41
41
|
* rows never materialize. Null/absent = current rows (the hot path).
|
|
42
42
|
*/
|
|
43
43
|
revisionAsOf?: number | null;
|
|
44
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Produces a missing or stale value and patches its row in `state`; absent = stored values
|
|
46
|
+
* only (production runtime).
|
|
47
|
+
*/
|
|
45
48
|
materialize?: (key: string) => Promise<JsonValue>;
|
|
46
49
|
/** Called with every key `get`/`load` touched, once the read is known — the reader's entity deps. */
|
|
47
50
|
onEntityRead?: (key: string) => void;
|
|
@@ -49,6 +52,11 @@ export type EntityStoreOptions = {
|
|
|
49
52
|
record?: RouteQueryRecorder;
|
|
50
53
|
};
|
|
51
54
|
export declare function createEntityStore(options: EntityStoreOptions): EntityStore;
|
|
55
|
+
/**
|
|
56
|
+
* Makes the state entry of `key` match its stored row after a materialization: in place, so
|
|
57
|
+
* every store over this state, and a `get` after a `load`, sees the loader's metadata.
|
|
58
|
+
*/
|
|
59
|
+
export declare function refreshStateEntity(database: ContentLayerDatabase, state: ContentStateView, key: string): Promise<void>;
|
|
52
60
|
/** The hash a hook's `entity` read records: identity, type, metadata, and value hash, or absence. */
|
|
53
61
|
export declare function hashEntityRead(state: ContentStateView, key: string): string;
|
|
54
62
|
/** Re-runs a stored `entity-type` query against `state`; the hash `list` recorded. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{hashJsonValue as e,toStableJsonString as t}from"./utils/json.js";import{compareStrings as n}from"../../shared/utils/compare-strings.js";import{entities as r}from"./schema.js";import{replaceDataRefMarkers as i}from"../materialization/data-ref.js";import{resolveDataRef as a}from"./blobs.js";import{listEntityRevisions as o}from"../revisions/list.js";import{selectEntityRowForRead as s}from"../revisions/read.js";import{and as c,eq as l,inArray as u}from"drizzle-orm";function d(e){let{database:t,state:u}=e,d=typeof e.revisionAsOf==`number`?e.revisionAsOf:null;return{get:async n=>{if(e.onEntityRead?.(n),d!==null){let e=await s(t.db,n,d);return e?x(e):null}let i=u?.entitiesByKey.get(n);if(i)return x(i);if(u)return null;let[a]=await t.db.select(b()).from(r).where(c(l(r.key,n),l(r.isCurrent,1))).limit(1);return a?x(a):null},async listRevisions(n){for(let t of n)e.onEntityRead?.(t);return o(t.db,n)},async list(i={}){let a=(u?[...u.entitiesByKey.values()].filter(e=>v(e,i)):await t.db.select(b()).from(r).where(c(l(r.isCurrent,1),i.type===void 0?void 0:l(r.type,i.type),i.pluginId===void 0?void 0:l(r.pluginId,i.pluginId)))).map(x).sort((e,t)=>n(e.key,t.key));return e.record?.({kind:`entity-type`,argument:m(i),resultHash:_(a.map(e=>e.key))}),a},async load(n){let o=u?.entitiesByKey.get(n),[
|
|
1
|
+
import{hashJsonValue as e,toStableJsonString as t}from"./utils/json.js";import{compareStrings as n}from"../../shared/utils/compare-strings.js";import{entities as r}from"./schema.js";import{replaceDataRefMarkers as i}from"../materialization/data-ref.js";import{resolveDataRef as a}from"./blobs.js";import{listEntityRevisions as o}from"../revisions/list.js";import{selectEntityRowForRead as s}from"../revisions/read.js";import{and as c,eq as l,inArray as u}from"drizzle-orm";function d(e){let{database:t,state:u}=e,d=typeof e.revisionAsOf==`number`?e.revisionAsOf:null;return{get:async n=>{if(e.onEntityRead?.(n),d!==null){let e=await s(t.db,n,d);return e?x(e):null}let i=u?.entitiesByKey.get(n);if(i)return x(i);if(u)return null;let[a]=await t.db.select(b()).from(r).where(c(l(r.key,n),l(r.isCurrent,1))).limit(1);return a?x(a):null},async listRevisions(n){for(let t of n)e.onEntityRead?.(t);return o(t.db,n)},async list(i={}){let a=(u?[...u.entitiesByKey.values()].filter(e=>v(e,i)):await t.db.select(b()).from(r).where(c(l(r.isCurrent,1),i.type===void 0?void 0:l(r.type,i.type),i.pluginId===void 0?void 0:l(r.pluginId,i.pluginId)))).map(x).sort((e,t)=>n(e.key,t.key));return e.record?.({kind:`entity-type`,argument:m(i),resultHash:_(a.map(e=>e.key))}),a},async load(n){let o=u?.entitiesByKey.get(n),[f]=d===null?u&&!o?[]:await t.db.select({status:r.status,data:r.data,dataHash:r.dataHash,type:r.type,metadata:r.metadata,isCurrent:r.isCurrent}).from(r).where(c(l(r.key,n),l(r.isCurrent,1))).limit(1):[await s(t.db,n,d)??void 0];if(!f)throw e.onEntityRead?.(n),Error(`Entity "${n}" is not registered.`);let m;if(f.status===`ready`&&f.data!==null||f.isCurrent!==1)m=f.data,u&&f.isCurrent===1&&d===null&&p(u,n,f);else if(e.materialize)m=await e.materialize(n);else throw Error(`Missing precomputed entity data for "${n}". Rebuild the content entity data before starting the production runtime.`);return e.onEntityRead?.(n),await i(m,e=>a(t,e))}}}async function f(e,t,n){let[i]=await e.db.select({status:r.status,dataHash:r.dataHash,type:r.type,metadata:r.metadata}).from(r).where(c(l(r.key,n),l(r.isCurrent,1))).limit(1);i&&p(t,n,i)}function p(e,t,n){let r=e.entitiesByKey.get(t);r&&Object.assign(r,{status:n.status,dataHash:n.dataHash,type:n.type,metadata:n.metadata})}function m(e){return t({...e.type===void 0?{}:{type:e.type},...e.pluginId===void 0?{}:{pluginId:e.pluginId}})}function h(t,n){let r=t.entitiesByKey.get(n);return e(r?[r.id,r.kind,r.type,r.metadata,r.dataHash,r.status]:`absent`)}function g(e,t){let r=JSON.parse(t),i=[];for(let t of e.entitiesByKey.values())v(t,r)&&i.push(t.key);return _(i.sort(n))}function _(t){return e([...t])}function v(e,t){return(t.type===void 0||e.type===t.type)&&(t.pluginId===void 0||e.pluginId===t.pluginId)}async function y(e,t){return t.length===0?[]:e.db.select({key:r.key,metadata:r.metadata}).from(r).where(c(u(r.key,[...t]),l(r.isCurrent,1)))}function b(){return{id:r.id,key:r.key,kind:r.kind,type:r.type,metadata:r.metadata,pluginId:r.pluginId,loaderId:r.loaderId,loaderArgs:r.loaderArgs,status:r.status,dataHash:r.dataHash,isSystem:r.isSystem}}function x(e){return{id:e.id,key:e.key,kind:e.kind,type:e.type,metadata:e.metadata,pluginId:e.pluginId,loaderId:e.loaderId,loaderArgs:e.loaderArgs,status:e.status,dataHash:e.dataHash,isSystem:e.isSystem===!0||e.isSystem===1}}export{d as createEntityStore,g as hashEntityListMembership,h as hashEntityRead,y as loadCurrentEntityMetadata,f as refreshStateEntity};
|
|
@@ -60,14 +60,12 @@ export declare function bulkSyncOrigins(database: ContentLayerDatabase, items: r
|
|
|
60
60
|
fallbackRoutes?: BulkSyncFallbackRoutes;
|
|
61
61
|
}): Promise<void>;
|
|
62
62
|
/**
|
|
63
|
-
*
|
|
64
|
-
* files match the revisions include globs
|
|
65
|
-
* (build/CLI on, dev watch off).
|
|
63
|
+
* The revision snapshots a capturing run cuts: `trackedKeys` holds the entity keys whose
|
|
64
|
+
* files match the revisions include globs. Absent when the run captures none.
|
|
66
65
|
*/
|
|
67
66
|
export type EntityUpsertRevisionsOptions = {
|
|
68
67
|
max: number;
|
|
69
68
|
trackedKeys: ReadonlySet<string>;
|
|
70
|
-
capture: boolean;
|
|
71
69
|
};
|
|
72
70
|
/**
|
|
73
71
|
* Upserts entity descriptors (deduped by key, last-wins), generating ids for
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{hashJsonValue as e}from"./utils/json.js";import{entities as t,entityDeps as n,filePlugins as r,linkRefs as i,originDeps as a,routeDeps as o,routes as s}from"./schema.js";import{batchValues as c,normalizeBatchSize as l,runWithSqliteBusyRetry as u}from"./connection.js";import{generateEntityId as d}from"./entities.js";import{garbageCollectOrphanedEntitiesInTransaction as f}from"./invalidation.js";import{deleteProblemsForFilesTx as p,deleteProblemsForPluginHookTx as m,insertProblemsTx as h,problemRow as g,replaceGlobalProblemsForKeysTx as _}from"./problems.js";import{deleteOriginLinkRefsForFilesTx as v}from"./link-refs.js";import{detectMarkdocPayloadFlag as y}from"../markdown/server/render.js";import{cutEntityRevisionTx as b}from"../revisions/cut.js";import{writePluginHookState as x}from"./plugin-hook-state.js";import{replaceRouteQueryDepsTx as S}from"./route-query-deps.js";import{and as C,eq as w,inArray as T,sql as E}from"drizzle-orm";async function D(e,t){let[r]=await e.select({originFileId:n.originFileId}).from(n).where(E`${n.entityId} = ${t.entityId} and ${n.dependencyType} = 'origin'`).limit(1);if(await e.delete(n).where(E`${n.entityId} = ${t.entityId} and ${n.dependencyType} in ('file','entity','glob')`),t.deps.length===0)return;if(!r&&!t.isSystem)throw Error(`replaceEntityRuntimeDeps: entity ${t.entityId} has no origin dep row.`);let i=/* @__PURE__ */ new Set,a=[];for(let e of t.deps){let n=e.kind===`file`?e.path:e.kind===`glob`?e.pattern:e.key,o=`${e.kind} ${n}`;i.has(o)||(i.add(o),a.push({entityId:t.entityId,originFileId:r?.originFileId??null,originPluginId:t.originPluginId,hook:`loader`,dependencyType:e.kind,dependencyRef:n}))}for(let t of c(a,500))await e.insert(n).values(t)}async function O(e,t,c){let d=c?.extractAllOrigins??[],p=c?.extractAllFailures??[],m=c?.globalProblems,v=c?.fallbackRoutes,y=(v?.removedSlugs.length??0)+(v?.added.length??0);if(t.length===0&&d.length===0&&p.length===0&&m===void 0&&y===0)return;let b=l(c?.batchSize);await u(()=>e.db.transaction(async e=>{await k(e,{fileIds:t.map(e=>e.fileId),extractAllOrigins:d,extractAllFailures:p,fallbackRoutes:v,batchSize:b}),await F(A(t),b,t=>e.insert(r).values(t));let l=j(t,d,v);await F(l.routes,b,t=>e.insert(s).values(t)),await F(l.routeDeps,b,t=>e.insert(o).values(t));let u=await I(e,l.entityDescriptors,b,c?.revisions);await F(l.originDeps,b,t=>e.insert(a).values(t)),await F(l.linkRefs,b,t=>e.insert(i).values(t));let y=[...l.problems,...p.map(e=>g(e.problem,{scope:`origin`,originFileId:null,originPluginId:e.pluginId,hook:`extractAll`}))];await h(e,y,b),m&&await _(e,m.source,m.keys,m.newProblems),l.entityDescriptors.length>0&&await F(P(l.entityOrigins,u),b,t=>e.insert(n).values(t));for(let t of d)await S(e,{originPluginId:t.originPluginId,hook:`extractAll`,entityId:null,dependencies:t.routeQueries}),await x(e,{pluginId:t.originPluginId,hook:`extractAll`,fingerprint:t.fingerprint});await f(e)}),{logger:c?.logger,operation:`bulk sync origins`})}async function k(e,t){for(let i of c(t.fileIds,t.batchSize))await e.delete(r).where(T(r.fileId,i)),await e.delete(o).where(T(o.originFileId,i)),await e.delete(s).where(T(s.sourceId,i)),await e.delete(n).where(T(n.originFileId,i)),await e.delete(a).where(T(a.originFileId,i));await p(e,t.fileIds,t.batchSize),await v(e,t.fileIds,t.batchSize),await m(e,[...t.extractAllOrigins.map(e=>e.originPluginId),...t.extractAllFailures.map(e=>e.pluginId)],`extractAll`);let i=t.extractAllOrigins.map(e=>e.originPluginId);for(let r of c(i,t.batchSize))await e.delete(o).where(E`${o.originFileId} is null and ${o.hook} = 'extractAll' and ${T(o.originPluginId,r)}`),await e.delete(s).where(E`${s.sourceId} is null and ${T(s.pluginId,r)}`),await e.delete(n).where(E`${n.originFileId} is null and ${n.hook} = 'extractAll' and ${T(n.originPluginId,r)}`),await e.delete(a).where(E`${a.originFileId} is null and ${a.hook} = 'extractAll' and ${T(a.originPluginId,r)}`);if(t.fallbackRoutes){let{pluginId:n,removedSlugs:r,added:i}=t.fallbackRoutes,a=[.../* @__PURE__ */ new Set([...r,...i.map(e=>e.slug)])];for(let r of c(a,t.batchSize))await e.delete(s).where(E`${s.sourceId} is null and ${s.pluginId} = ${n} and ${T(s.slug,r)}`)}}function A(e){let t=[];for(let n of e)for(let e of n.plugins)t.push({fileId:n.fileId,pluginId:e.id,pluginVersion:e.version});return t}function j(e,t,n){let r={routes:[],routeDeps:[],originDeps:[],entityDescriptors:[],entityOrigins:[],problems:[],linkRefs:[]};for(let t of e)for(let e of t.origins)N(e,r);for(let e of t)N(e,r);for(let e of n?.added??[])r.routes.push(M(e,null));return r}function M(e,t){return{metadata:e.metadata,pluginId:e.pluginId,routeId:e.routeId,sidebarSubtreeEntityKey:e.sidebarSubtreeEntityKey,wildcardPrefix:e.wildcardPrefix??!1,slug:e.slug,requestedSlug:e.requestedSlug,slugType:e.slugType,sourceId:t,template:e.template,versionGroup:e.versionGroup,version:e.version,locale:e.locale,isFallback:e.isFallback,canonicalSlug:e.canonicalSlug}}function N(e,t){for(let n of e.routes)t.routes.push(M(n,e.originFileId));for(let n of e.routeDependencies)t.routeDeps.push({args:n.args,dependencyType:n.dependencyType,key:n.key,originFileId:e.originFileId,originPluginId:e.originPluginId,hook:e.hook,routeSlug:n.routeSlug,target:n.target});for(let n of e.originDependencies)t.originDeps.push({accessType:n.accessType,originFileId:e.originFileId,originPluginId:e.originPluginId,hook:e.hook,source:n.source,path:n.path,valueHash:n.valueHash});for(let n of e.problems)t.problems.push(g(n,{scope:`origin`,originFileId:e.originFileId,originPluginId:e.originPluginId,hook:e.hook}));e.entities.length>0&&(t.entityDescriptors.push(...e.entities),t.entityOrigins.push({originFileId:e.originFileId,originFilePath:e.originFilePath,originPluginId:e.originPluginId,hook:e.hook,descriptors:e.entities}));for(let n of e.linkRefs??[]){let r=n.filePath??e.originFilePath;r!==null&&t.linkRefs.push({entityId:null,originFileId:e.originFileId,originPluginId:e.originPluginId,filePath:r,kind:n.kind,raw:n.raw,targetKind:n.targetKind,target:n.target,fragment:n.fragment,title:n.title??null,range:n.range??null})}}function P(e,t){let n=[];for(let r of e)for(let e of r.descriptors){let i=t.get(e.key);if(!i)throw Error(`bulkSyncOrigins: missing entity id for key "${e.key}" after upsert.`);n.push({dependencyRef:r.originFilePath??r.originPluginId,dependencyType:`origin`,entityId:i,originFileId:r.originFileId,originPluginId:r.originPluginId,hook:r.hook})}return n}async function F(e,t,n){for(let r of c(e,t))r.length>0&&await n(r)}async function I(n,r,i=500,a){if(r.length===0)return/* @__PURE__ */ new Map;let o=/* @__PURE__ */ new Map;for(let e of r)o.set(e.key,e);let s=[...o.keys()],l=/* @__PURE__ */ new Map,u=/* @__PURE__ */ new Map;for(let e of c(s,i)){let r=await n.select({id:t.id,key:t.key,kind:t.kind,type:t.type,metadata:t.metadata,pluginId:t.pluginId,loaderId:t.loaderId,loaderArgs:t.loaderArgs,data:t.data,markdoc:t.markdoc,dataHash:t.dataHash,version:t.version,revisionCreatedAt:t.revisionCreatedAt}).from(t).where(C(T(t.key,e),w(t.isCurrent,1)));for(let e of r)l.set(e.key,e.id),u.set(e.key,e)}let f=Date.now(),p=[];for(let t of o.values()){let r=u.get(t.key),i=r?.id??d();l.set(t.key,i);let o=t.kind===`inline`?`ready`:t.shouldInvalidateValue||!r?`stale`:`ready`,s=t.kind===`inline`?t.value:null,c=s===null?null:e(s),m=r?.revisionCreatedAt??f;t.kind===`inline`&&r&&(r.dataHash??null)!==c&&(a?.
|
|
1
|
+
import{hashJsonValue as e}from"./utils/json.js";import{entities as t,entityDeps as n,filePlugins as r,linkRefs as i,originDeps as a,routeDeps as o,routes as s}from"./schema.js";import{batchValues as c,normalizeBatchSize as l,runWithSqliteBusyRetry as u}from"./connection.js";import{generateEntityId as d}from"./entities.js";import{garbageCollectOrphanedEntitiesInTransaction as f}from"./invalidation.js";import{deleteProblemsForFilesTx as p,deleteProblemsForPluginHookTx as m,insertProblemsTx as h,problemRow as g,replaceGlobalProblemsForKeysTx as _}from"./problems.js";import{deleteOriginLinkRefsForFilesTx as v}from"./link-refs.js";import{detectMarkdocPayloadFlag as y}from"../markdown/server/render.js";import{cutEntityRevisionTx as b}from"../revisions/cut.js";import{writePluginHookState as x}from"./plugin-hook-state.js";import{replaceRouteQueryDepsTx as S}from"./route-query-deps.js";import{and as C,eq as w,inArray as T,sql as E}from"drizzle-orm";async function D(e,t){let[r]=await e.select({originFileId:n.originFileId}).from(n).where(E`${n.entityId} = ${t.entityId} and ${n.dependencyType} = 'origin'`).limit(1);if(await e.delete(n).where(E`${n.entityId} = ${t.entityId} and ${n.dependencyType} in ('file','entity','glob')`),t.deps.length===0)return;if(!r&&!t.isSystem)throw Error(`replaceEntityRuntimeDeps: entity ${t.entityId} has no origin dep row.`);let i=/* @__PURE__ */ new Set,a=[];for(let e of t.deps){let n=e.kind===`file`?e.path:e.kind===`glob`?e.pattern:e.key,o=`${e.kind} ${n}`;i.has(o)||(i.add(o),a.push({entityId:t.entityId,originFileId:r?.originFileId??null,originPluginId:t.originPluginId,hook:`loader`,dependencyType:e.kind,dependencyRef:n}))}for(let t of c(a,500))await e.insert(n).values(t)}async function O(e,t,c){let d=c?.extractAllOrigins??[],p=c?.extractAllFailures??[],m=c?.globalProblems,v=c?.fallbackRoutes,y=(v?.removedSlugs.length??0)+(v?.added.length??0);if(t.length===0&&d.length===0&&p.length===0&&m===void 0&&y===0)return;let b=l(c?.batchSize);await u(()=>e.db.transaction(async e=>{await k(e,{fileIds:t.map(e=>e.fileId),extractAllOrigins:d,extractAllFailures:p,fallbackRoutes:v,batchSize:b}),await F(A(t),b,t=>e.insert(r).values(t));let l=j(t,d,v);await F(l.routes,b,t=>e.insert(s).values(t)),await F(l.routeDeps,b,t=>e.insert(o).values(t));let u=await I(e,l.entityDescriptors,b,c?.revisions);await F(l.originDeps,b,t=>e.insert(a).values(t)),await F(l.linkRefs,b,t=>e.insert(i).values(t));let y=[...l.problems,...p.map(e=>g(e.problem,{scope:`origin`,originFileId:null,originPluginId:e.pluginId,hook:`extractAll`}))];await h(e,y,b),m&&await _(e,m.source,m.keys,m.newProblems),l.entityDescriptors.length>0&&await F(P(l.entityOrigins,u),b,t=>e.insert(n).values(t));for(let t of d)await S(e,{originPluginId:t.originPluginId,hook:`extractAll`,entityId:null,dependencies:t.routeQueries}),await x(e,{pluginId:t.originPluginId,hook:`extractAll`,fingerprint:t.fingerprint});await f(e)}),{logger:c?.logger,operation:`bulk sync origins`})}async function k(e,t){for(let i of c(t.fileIds,t.batchSize))await e.delete(r).where(T(r.fileId,i)),await e.delete(o).where(T(o.originFileId,i)),await e.delete(s).where(T(s.sourceId,i)),await e.delete(n).where(T(n.originFileId,i)),await e.delete(a).where(T(a.originFileId,i));await p(e,t.fileIds,t.batchSize),await v(e,t.fileIds,t.batchSize),await m(e,[...t.extractAllOrigins.map(e=>e.originPluginId),...t.extractAllFailures.map(e=>e.pluginId)],`extractAll`);let i=t.extractAllOrigins.map(e=>e.originPluginId);for(let r of c(i,t.batchSize))await e.delete(o).where(E`${o.originFileId} is null and ${o.hook} = 'extractAll' and ${T(o.originPluginId,r)}`),await e.delete(s).where(E`${s.sourceId} is null and ${T(s.pluginId,r)}`),await e.delete(n).where(E`${n.originFileId} is null and ${n.hook} = 'extractAll' and ${T(n.originPluginId,r)}`),await e.delete(a).where(E`${a.originFileId} is null and ${a.hook} = 'extractAll' and ${T(a.originPluginId,r)}`);if(t.fallbackRoutes){let{pluginId:n,removedSlugs:r,added:i}=t.fallbackRoutes,a=[.../* @__PURE__ */ new Set([...r,...i.map(e=>e.slug)])];for(let r of c(a,t.batchSize))await e.delete(s).where(E`${s.sourceId} is null and ${s.pluginId} = ${n} and ${T(s.slug,r)}`)}}function A(e){let t=[];for(let n of e)for(let e of n.plugins)t.push({fileId:n.fileId,pluginId:e.id,pluginVersion:e.version});return t}function j(e,t,n){let r={routes:[],routeDeps:[],originDeps:[],entityDescriptors:[],entityOrigins:[],problems:[],linkRefs:[]};for(let t of e)for(let e of t.origins)N(e,r);for(let e of t)N(e,r);for(let e of n?.added??[])r.routes.push(M(e,null));return r}function M(e,t){return{metadata:e.metadata,pluginId:e.pluginId,routeId:e.routeId,sidebarSubtreeEntityKey:e.sidebarSubtreeEntityKey,wildcardPrefix:e.wildcardPrefix??!1,slug:e.slug,requestedSlug:e.requestedSlug,slugType:e.slugType,sourceId:t,template:e.template,versionGroup:e.versionGroup,version:e.version,locale:e.locale,isFallback:e.isFallback,canonicalSlug:e.canonicalSlug}}function N(e,t){for(let n of e.routes)t.routes.push(M(n,e.originFileId));for(let n of e.routeDependencies)t.routeDeps.push({args:n.args,dependencyType:n.dependencyType,key:n.key,originFileId:e.originFileId,originPluginId:e.originPluginId,hook:e.hook,routeSlug:n.routeSlug,target:n.target});for(let n of e.originDependencies)t.originDeps.push({accessType:n.accessType,originFileId:e.originFileId,originPluginId:e.originPluginId,hook:e.hook,source:n.source,path:n.path,valueHash:n.valueHash});for(let n of e.problems)t.problems.push(g(n,{scope:`origin`,originFileId:e.originFileId,originPluginId:e.originPluginId,hook:e.hook}));e.entities.length>0&&(t.entityDescriptors.push(...e.entities),t.entityOrigins.push({originFileId:e.originFileId,originFilePath:e.originFilePath,originPluginId:e.originPluginId,hook:e.hook,descriptors:e.entities}));for(let n of e.linkRefs??[]){let r=n.filePath??e.originFilePath;r!==null&&t.linkRefs.push({entityId:null,originFileId:e.originFileId,originPluginId:e.originPluginId,filePath:r,kind:n.kind,raw:n.raw,targetKind:n.targetKind,target:n.target,fragment:n.fragment,title:n.title??null,range:n.range??null})}}function P(e,t){let n=[];for(let r of e)for(let e of r.descriptors){let i=t.get(e.key);if(!i)throw Error(`bulkSyncOrigins: missing entity id for key "${e.key}" after upsert.`);n.push({dependencyRef:r.originFilePath??r.originPluginId,dependencyType:`origin`,entityId:i,originFileId:r.originFileId,originPluginId:r.originPluginId,hook:r.hook})}return n}async function F(e,t,n){for(let r of c(e,t))r.length>0&&await n(r)}async function I(n,r,i=500,a){if(r.length===0)return/* @__PURE__ */ new Map;let o=/* @__PURE__ */ new Map;for(let e of r)o.set(e.key,e);let s=[...o.keys()],l=/* @__PURE__ */ new Map,u=/* @__PURE__ */ new Map;for(let e of c(s,i)){let r=await n.select({id:t.id,key:t.key,kind:t.kind,type:t.type,metadata:t.metadata,pluginId:t.pluginId,loaderId:t.loaderId,loaderArgs:t.loaderArgs,data:t.data,markdoc:t.markdoc,dataHash:t.dataHash,version:t.version,revisionCreatedAt:t.revisionCreatedAt}).from(t).where(C(T(t.key,e),w(t.isCurrent,1)));for(let e of r)l.set(e.key,e.id),u.set(e.key,e)}let f=Date.now(),p=[];for(let t of o.values()){let r=u.get(t.key),i=r?.id??d();l.set(t.key,i);let o=t.kind===`inline`?`ready`:t.shouldInvalidateValue||!r?`stale`:`ready`,s=t.kind===`inline`?t.value:null,c=s===null?null:e(s),m=r?.revisionCreatedAt??f;t.kind===`inline`&&r&&(r.dataHash??null)!==c&&(a?.trackedKeys.has(t.key)&&await b(n,{current:r,max:a.max}),m=Math.max(f,r.revisionCreatedAt+1)),p.push({id:i,key:t.key,kind:t.kind,type:t.type,metadata:t.metadata,pluginId:t.pluginId,loaderId:t.loaderId,loaderArgs:t.loaderArgs,status:o,data:s,markdoc:s===null?null:y(JSON.stringify(s)),dataHash:c,isSystem:+!!t.isSystem,version:t.version,isCurrent:1,revisionCreatedAt:m})}for(let e of c(p,i))e.length!==0&&await n.insert(t).values(e).onConflictDoUpdate({target:t.id,set:{kind:E.raw(`excluded.kind`),type:E.raw(`excluded.type`),metadata:E.raw(`CASE WHEN excluded.kind = 'inline' OR excluded.status = 'stale' THEN excluded.metadata ELSE entities.metadata END`),pluginId:E.raw(`excluded.plugin_id`),loaderId:E.raw(`excluded.loader_id`),loaderArgs:E.raw(`excluded.loader_args`),status:E.raw(`CASE WHEN excluded.kind = 'inline' OR excluded.status = 'stale' THEN excluded.status ELSE entities.status END`),data:E.raw(`CASE WHEN excluded.kind = 'inline' THEN excluded.data ELSE entities.data END`),markdoc:E.raw(`CASE WHEN excluded.kind = 'inline' THEN excluded.markdoc ELSE entities.markdoc END`),dataHash:E.raw(`CASE WHEN excluded.kind = 'inline' THEN excluded.data_hash ELSE entities.data_hash END`),isSystem:E.raw(`excluded.is_system`),version:E.raw(`excluded.version`),revisionCreatedAt:E.raw(`excluded.revision_created_at`),invalidationSeq:E`case when excluded.status = 'stale' then ${t.invalidationSeq} + 1 else ${t.invalidationSeq} end`}});return l}export{O as bulkSyncOrigins,D as replaceEntityRuntimeDeps,I as upsertEntityDescriptors};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{createRecordingPluginIO as e}from"../plugins/io.js";import{createTrackedRedoclyConfig as t}from"../config-deps.js";import{dedupeFileDependencies as n}from"../plugins/execute.js";import{dedupeRouteQueryDependencies as r}from"../plugins/utils/route-index.js";import"../plugins/index.js";import{extractAllFailedProblem as i}from"../problems/producers.js";import{buildHookSourceFiles as a,createHookEntityStore as o,resolvePluginHookTriggers as s}from"./hook-inputs.js";import{routeKeepsSlug as c}from"./slugger/drop-colliding-routes.js";import{normalizePluginArtifacts as l}from"./normalize-artifacts.js";import{applyOriginRoutesToWorkingState as u,removeOriginFromWorkingState as d}from"./working-state.js";async function f(c){let f=[],p=[],m=c.context.pluginRegistry.listExtractAllPlugins();if(
|
|
1
|
+
import{createRecordingPluginIO as e}from"../plugins/io.js";import{createTrackedRedoclyConfig as t}from"../config-deps.js";import{dedupeFileDependencies as n}from"../plugins/execute.js";import{dedupeRouteQueryDependencies as r}from"../plugins/utils/route-index.js";import"../plugins/index.js";import{extractAllFailedProblem as i}from"../problems/producers.js";import{buildHookSourceFiles as a,createHookEntityStore as o,resolvePluginHookTriggers as s}from"./hook-inputs.js";import{routeKeepsSlug as c}from"./slugger/drop-colliding-routes.js";import{normalizePluginArtifacts as l}from"./normalize-artifacts.js";import{applyOriginRoutesToWorkingState as u,removeOriginFromWorkingState as d}from"./working-state.js";async function f(c){let f=[],p=[],m=c.materializer.state,h=c.context.pluginRegistry.listExtractAllPlugins();if(h.length===0)return{pending:f,failures:p};let g=s(`extractAll`,c.context,m,c.originRows,c.routeQueryRows,c.hookState),_=c.context.pluginRegistry.list().map(e=>e.id);for(let s of h){if(!g.shouldRun(s)||!s.content?.extractAll)continue;let h=c.logger.child(`content-layer:plugin:${s.id}`),v=t(c.context.config),y=[],b=[],x=e=>b.push(e),S=await a(g.matchedPathsOf(s),c.files,{database:c.database,workingState:m}),C;try{C=await s.content.extractAll({files:S,io:e(c.context.platform.fs,c.context.contentDir,()=>m.filesByPath.keys(),e=>y.push(e)),config:v.config,entities:o(c.database,c.materializer,x),logger:h})}catch(e){let t=e instanceof Error?e:Error(String(e));h.warn(`Content plugin extractAll failed`,{plugin:s.id,error:t.message}),p.push({pluginId:s.id,problem:i({pluginId:s.id,message:t.message})});continue}let w=l({context:c.context,filePath:null,pluginId:s.id,routeDrafts:C?.routes??[],standaloneEntities:C?.entities??[]});d(m,_,s.id,null),u(m,_,s.id,null,w.routes),f.push({pluginId:s.id,inputsHash:g.fingerprintOf(s),routes:w.routes,routeDependencies:w.routeDependencies,entities:w.entities,originDependencies:[...v.collectDependencies().map(e=>Object.assign(e,{source:`config`})),...n(y)],routeQueries:r(b)})}return{pending:f,failures:p}}function p(e,t){if(t.size!==0)for(let n of e){n.routes=n.routes.filter(e=>c(t,n.pluginId,null,e));let e=new Set(n.routes.map(e=>`${e.routeId}\u0000${e.requestedSlug}`));n.routeDependencies=n.routeDependencies.filter(t=>e.has(`${t.routeId}\u0000${t.routeSlug}`))}}export{p as dropCollidingExtractAllRoutes,f as runExtractAllPhase};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{hashJsonValue as e,isJsonObject as t,toJsonObject as n}from"../database/utils/json.js";import{compareStrings as r}from"../../shared/utils/compare-strings.js";import{createRecordingPluginIO as i}from"../plugins/io.js";import{createTrackedRedoclyConfig as a}from"../config-deps.js";import{dedupeFileDependencies as o,runContentPluginFinalize as s}from"../plugins/execute.js";import{dedupeRouteQueryDependencies as c}from"../plugins/utils/route-index.js";import"../../shared/constants/app-data.js";import{appDataSystemEntityDescriptor as l,isAppDataFragment as u}from"../plugins/internal/app-data/index.js";import"../plugins/index.js";import{createProblem as d}from"../problems/codes.js";import{propagateEntityInvalidationFromKeys as f}from"../database/invalidation.js";import{listGlobalProblems as p,problemRow as m}from"../database/problems.js";import{listStaticFileRecords as h,replaceStaticFileRows as g}from"../database/static-files.js";import{createSearchHandle as _}from"../search/store.js";import{appDataNamespaceConflictProblem as v,attachConflictProblem as y,finalizeFailedProblem as b,keysBlamingPlugins as x}from"../problems/producers.js";import{buildHookSourceFiles as S,createHookEntityStore as C,finalizeHookInputs as w,resolvePluginHookTriggers as T}from"./hook-inputs.js";import{loadPluginScopedRouteDepOwners as E,registerSystemEntities as D,replaceFinalizeArtifacts as O,replaceFinalizeProblems as k,replaceFinalizeReads as A}from"../database/finalize.js";import{createEntityDescriber as j,normalizeRouteDependency as M}from"./normalize-artifacts.js";import{collectStaticFileRows as N,settleStaticFileClaims as P}from"../static/registry.js";async function F(e){let t=T(`finalize`,e.context,
|
|
1
|
+
import{hashJsonValue as e,isJsonObject as t,toJsonObject as n}from"../database/utils/json.js";import{compareStrings as r}from"../../shared/utils/compare-strings.js";import{createRecordingPluginIO as i}from"../plugins/io.js";import{createTrackedRedoclyConfig as a}from"../config-deps.js";import{dedupeFileDependencies as o,runContentPluginFinalize as s}from"../plugins/execute.js";import{dedupeRouteQueryDependencies as c}from"../plugins/utils/route-index.js";import"../../shared/constants/app-data.js";import{appDataSystemEntityDescriptor as l,isAppDataFragment as u}from"../plugins/internal/app-data/index.js";import"../plugins/index.js";import{createProblem as d}from"../problems/codes.js";import{propagateEntityInvalidationFromKeys as f}from"../database/invalidation.js";import{listGlobalProblems as p,problemRow as m}from"../database/problems.js";import{listStaticFileRecords as h,replaceStaticFileRows as g}from"../database/static-files.js";import{createSearchHandle as _}from"../search/store.js";import{appDataNamespaceConflictProblem as v,attachConflictProblem as y,finalizeFailedProblem as b,keysBlamingPlugins as x}from"../problems/producers.js";import{buildHookSourceFiles as S,createHookEntityStore as C,finalizeHookInputs as w,resolvePluginHookTriggers as T}from"./hook-inputs.js";import{loadPluginScopedRouteDepOwners as E,registerSystemEntities as D,replaceFinalizeArtifacts as O,replaceFinalizeProblems as k,replaceFinalizeReads as A}from"../database/finalize.js";import{createEntityDescriber as j,normalizeRouteDependency as M}from"./normalize-artifacts.js";import{collectStaticFileRows as N,settleStaticFileClaims as P}from"../static/registry.js";async function F(e){let t=e.materializer.state,n=T(`finalize`,e.context,t,e.pluginScopedOriginRows,e.routeQueryRows,e.hookState,w(e.localeContext)),r=e.context.pluginRegistry.listFinalizePlugins().filter(n.shouldRun).map(e=>({plugin:e,inputsHash:n.fingerprintOf(e)})),i=r.map(({plugin:e})=>e.id),a=await L(e.database,i),o=/* @__PURE__ */ new Map,s=R(t,e.context),c=/* @__PURE__ */ new Map,l=[],u=[],d=[],f=[];for(let{plugin:t,inputsHash:i}of r){let r=await I({plugin:t,context:e.context,database:e.database,localeContext:e.localeContext,materializer:e.materializer,processedFiles:e.files,matchedPaths:n.matchedPathsOf(t),routeQueries:n.evaluator,attachOwnerByKey:a,staticFilesByPlugin:o,namespaceOwners:s,fragmentUpdates:c,pendingTrackedDependencyWrites:l,inputsHash:i,logger:e.logger,collectEntities:e.collectEntities,conflicts:d,staticProblems:f});for(let e of r.problems)u.push({pluginId:t.id,problem:e})}c.size>0&&await q(e.database,e.context,c);let h=new Set(e.context.pluginRegistry.listFinalizePlugins().map(e=>e.id));if(await W({context:e.context,database:e.database,workingState:t},o,f,h),i.length>0){let t=await p(e.database,`finalize`),n=U(t,new Set(i),h);await k(e.database,{pluginIds:i,pluginProblems:u.map(({pluginId:e,problem:t})=>m(t,{scope:`origin`,originFileId:null,originPluginId:e,hook:`finalize`})),globalKeys:[.../* @__PURE__ */ new Set([...d.map(e=>e.key),...x(t,n)])],globalProblems:d.map(e=>m(e.problem,{scope:`global`,source:`finalize`,key:e.key})),logger:e.logger})}for(let t of l)await V(e.database,t);return{ranPluginIds:i}}async function I(e){let t=e.logger.child(`content-layer:plugin:${e.plugin.id}`),r=e.materializer.state,o=await S(e.matchedPaths,e.processedFiles,{database:e.database,workingState:r}),c=a(e.context.config),l=[],u=i(e.context.platform.fs,e.context.contentDir,()=>r.filesByPath.keys(),e=>l.push(e)),f=[],p=e=>f.push(e),m=await s(e.plugin,{files:o,routes:e.routeQueries.index(p),io:u,config:c.config,localeContext:e.localeContext,entities:C(e.database,e.materializer,p),search:_(e.database),logger:t});if(!m.ok)return{problems:[b({pluginId:e.plugin.id,message:m.error.message})]};let h=m.value??{};return e.fragmentUpdates.set(e.plugin.id,h.appData===void 0?B(e.plugin.id,e.namespaceOwners):z({pluginId:e.plugin.id,appData:n(h.appData),namespaceOwners:e.namespaceOwners,conflicts:e.conflicts})),e.staticFilesByPlugin.set(e.plugin.id,await H({plugin:e.plugin,result:h,workingState:r,context:e.context,database:e.database,attachOwnerByKey:e.attachOwnerByKey,recordFileDependency:e=>l.push(e),conflicts:e.conflicts,staticProblems:e.staticProblems,logger:t,collectEntities:e.collectEntities})),e.pendingTrackedDependencyWrites.push({pluginId:e.plugin.id,inputsHash:e.inputsHash,configDependencies:c.collectDependencies(),fileDependencies:l,routeQueryDependencies:f,logger:t}),{problems:(h.problems??[]).map(d)}}async function L(e,t){let n=await E(e,t);return new Map(n.map(e=>[`${e.routeSlug}\u0000${e.key}`,e.originPluginId]))}function R(e,n){let r=/* @__PURE__ */ new Map,i=e.entitiesByKey.get(`app-data`)?.loaderArgs??null,a=t(i)&&t(i.fragments)?i.fragments:{},o=new Set(n.pluginRegistry.listFinalizePlugins().map(e=>e.id));for(let[e,t]of Object.entries(a))u(t)&&o.has(t.owner)&&r.set(e,t.owner);return r}function z(e){B(e.pluginId,e.namespaceOwners);let t={};for(let[n,r]of Object.entries(e.appData)){let i=e.namespaceOwners.get(n);if(i!==void 0&&i!==e.pluginId){e.conflicts.push(v({namespace:n,ownerPluginId:i,pluginId:e.pluginId}));continue}e.namespaceOwners.set(n,e.pluginId),t[n]=r}return t}function B(e,t){for(let[n,r]of t)r===e&&t.delete(n);return null}async function V(e,t){let n=[...t.configDependencies.map(e=>({...e,source:`config`})),...o(t.fileDependencies)];await A(e,{originPluginId:t.pluginId,fingerprint:t.inputsHash,dependencies:n,routeQueries:c(t.routeQueryDependencies),logger:t.logger})}async function H(t){let n=G({context:t.context,pluginId:t.plugin.id,result:t.result,state:t.workingState,attachOwnerByKey:t.attachOwnerByKey,recordFileDependency:t.recordFileDependency,conflicts:t.conflicts,staticProblems:t.staticProblems,logger:t.logger});await O(t.database,{originPluginId:t.plugin.id,routeDependencies:n.routeDependencies,entities:n.entities,logger:t.logger}),t.collectEntities?.(n.entities);let r=n.entities.flatMap(({key:n,kind:r,value:i})=>r===`inline`&&t.workingState.entitiesByKey.get(n)?.dataHash!==(i===null?null:e(i))?[n]:[]);if(r.length>0){let{invalidatedKeys:e}=await f(t.database,{keys:r,reason:`invalidate finalize entity readers`,markKeys:!1,logger:t.logger});for(let n of e){let e=t.workingState.entitiesByKey.get(n);e&&(e.status=`stale`)}}return n.staticFiles}function U(e,t,n){let r=new Set(t);for(let t of e){let e=t.data===null?void 0:t.data.pluginId;typeof e==`string`&&!n.has(e)&&r.add(e)}return r}async function W(e,t,n,r){let i=await h(e.database),a=await p(e.database,`static`),o=i.filter(e=>!t.has(e.ownerPluginId)),s=P({kept:o,collected:[...t],routeBySlug:e.workingState.routeBySlug,reservedSlugs:e.context.pluginRegistry.listReservedSlugs(),problems:n}),c=U(a,new Set(t.keys()),r),l=/* @__PURE__ */ new Set([...n.map(e=>e.key),...o.map(e=>e.path),...x(a,c)]);await g(e.database,{ownerPluginIds:[...s.keys()],rows:[...s].flatMap(([e,t])=>t.map(t=>({...t,ownerPluginId:e}))),problems:{keys:[...l],newProblems:n.map(e=>m(e.problem,{scope:`global`,source:`static`,key:e.key}))}})}function G(e){let t=/* @__PURE__ */ new Map,n=[],i=j(e.context,e.pluginId),a=e=>{t.set(e.key,e)};for(let t of e.result.entities??[])a(i(t));for(let t of e.result.attach??[])K({attachment:t,describeEntity:i,pluginId:e.pluginId,state:e.state,attachOwnerByKey:e.attachOwnerByKey,conflicts:e.conflicts,registerEntity:a,routeDependencies:n});let o=N({pluginId:e.pluginId,declarations:e.result.staticFiles??[],filesByPath:e.state.filesByPath,registerProducer:e=>{let t={...i(e.producer),shouldInvalidateValue:!1};return a(t),t.key},recordFileDependency:e.recordFileDependency,problems:e.staticProblems});return{routeDependencies:n,entities:[...t.values()].sort((e,t)=>r(e.key,t.key)),staticFiles:o}}function K(e){let{attachment:t}=e;if(e.state.routeBySlug.get(t.slug))for(let[n,r]of Object.entries(t.deps)){let i=`${t.slug}\u0000${n}`,a=e.attachOwnerByKey.get(i);if(a!==void 0){a!==e.pluginId&&e.conflicts.push(y({slug:t.slug,key:n,ownerPluginId:a,pluginId:e.pluginId}));continue}e.attachOwnerByKey.set(i,e.pluginId),e.routeDependencies.push(M({dependency:r,describeEntity:e.describeEntity,key:n,registerEntity:e.registerEntity,routeSlug:t.slug}))}}async function q(e,t,n){await D(e,[l({fragmentUpdates:n})],{logger:t.logger.child(`content-layer:db`)})}export{U as claimantsToReplace,F as runFinalizePhase};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{hashJsonValue as e}from"../database/utils/json.js";import{compareStrings as t}from"../../shared/utils/compare-strings.js";import{matchesContentPluginFilter as n}from"../plugins/registry.js";import{createSourceFile as r}from"../plugins/source-file.js";import{createEntityStore as i,hashEntityRead as a}from"../database/entity-store.js";import{createRouteQueryEvaluator as o,isRouteQueryStale as s}from"../plugins/utils/route-index.js";import"../plugins/index.js";import{computeDirtyPluginScopedOrigins as c}from"../database/invalidation.js";import{loadFileManifestsByPaths as l}from"../database/files.js";
|
|
1
|
+
import{hashJsonValue as e}from"../database/utils/json.js";import{compareStrings as t}from"../../shared/utils/compare-strings.js";import{matchesContentPluginFilter as n}from"../plugins/registry.js";import{createSourceFile as r}from"../plugins/source-file.js";import{createEntityStore as i,hashEntityRead as a}from"../database/entity-store.js";import{createRouteQueryEvaluator as o,isRouteQueryStale as s}from"../plugins/utils/route-index.js";import"../plugins/index.js";import{computeDirtyPluginScopedOrigins as c}from"../database/invalidation.js";import{loadFileManifestsByPaths as l}from"../database/files.js";function u(e){return[e.defaultLocale,[...e.locales]]}function d(t,n,r,i,a,l,u){let d=new Set(c(i,t,{config:n.config,filesByPath:r.filesByPath,ignoredFilesByPath:r.ignoredFilesByPath})),f=o(r,n.config),m=f.memoized();for(let e of a)e.hook===t&&e.entityId===null&&!d.has(e.originPluginId)&&s(m,e)&&d.add(e.originPluginId);let h=/* @__PURE__ */ new Map,g=e=>{let t=h.get(e.id);return t||(t=p(e,r.filesByPath),h.set(e.id,t)),t},_=t=>e(u===void 0?[t.version,g(t)]:[t.version,g(t),u]);return{evaluator:f,fingerprintOf:_,matchedPathsOf:e=>g(e).map(([e])=>e),shouldRun:e=>d.has(e.id)||l[t].get(e.id)!==_(e)}}function f(e,t,n,r,i,a,o){let s=d(e,t,n,r,i,a,o);return(e===`extractAll`?t.pluginRegistry.listExtractAllPlugins():t.pluginRegistry.listFinalizePlugins()).some(s.shouldRun)}function p(e,r){let i=[];for(let[t,a]of r)n(e.content?.filter,t)&&i.push([t,a.hash]);return i.sort((e,n)=>t(e[0],n[0]))}async function m(e,t,n){let i=new Map(t.map(e=>[e.path,e])),a=e.filter(e=>i.get(e)?.manifest===void 0),o=a.length>0?await l(n.database,a):/* @__PURE__ */ new Map,s=[];for(let t of e){let e=n.workingState.filesByPath.get(t);if(!e)continue;let a=i.get(t)?.manifest??o.get(t)??{};s.push(r({path:t,hash:e.hash,manifest:a}))}return s}function h(e,t,n){let{state:r}=t;return i({database:e,state:r,materialize:t.materialize,onEntityRead:e=>n({kind:`entity`,argument:e,resultHash:a(r,e)}),record:n})}export{m as buildHookSourceFiles,h as createHookEntityStore,u as finalizeHookInputs,f as hasAnyPluginHookTrigger,d as resolvePluginHookTriggers};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{compareStrings as e}from"../../shared/utils/compare-strings.js";import{L10N_DIR as t,TRANSLATIONS_FILE_NAMES as n,TRANSLATIONS_FILE_PATTERN as r}from"../../shared/constants/l10n/defaults.js";import{isObject as i}from"../../shared/utils/is-object.js";import{slugifySegment as a}from"../../shared/utils/routes/slugify-segment.js";import{collectL10nDirLocales as o,isLocaleCode as s,l10nDirOf as c,normalizeL10nConfig as l}from"../../shared/utils/l10n.js";import{createProblem as u}from"../problems/codes.js";function d(e){if(e==null)return[];if(!i(e))return[f("Invalid `l10n` config: expected an object")];let t=[],{defaultLocale:n,locales:r}=e,a=n!=null;if(a&&!s(n)&&t.push(f("Invalid `l10n.defaultLocale`: expected a non-empty string")),r==null)return t;if(!Array.isArray(r))return t.push(f("Invalid `l10n.locales`: expected an array")),t;for(let e of r)(!i(e)||!s(e.code))&&t.push(f("Invalid `l10n.locales` entry: expected an object with a string `code`"));return a?s(n)&&!l(e).locales?.some(e=>e.code===n)&&t.push(f(`Invalid \`l10n.locales\`: the default locale \`${n}\` must be listed`)):t.push(f("Invalid `l10n`: `locales` requires `defaultLocale`")),t}function f(e){return u({code:`configuration/l10n-invalid`,message:e})}function p(e){let{localeContext:t}=e,n=[...e.filePaths],r=o(n),i=t.locales,s=[];r.includes(t.defaultLocale)&&s.push({kind:`default-locale-dir`,dir:t.defaultLocale});let u=n.filter(e=>e.startsWith(`@l10n/`)&&c(e)===void 0);u.length>0&&s.push({kind:`stray-file`,paths:u.sort()});let d=/* @__PURE__ */ new Map;for(let e of i){let t=a(e);d.set(t,[...d.get(t)??[],e])}for(let e of d.values())e.length>1&&s.push({kind:`dir-slug-collision`,codes:e});s.push(...m(n,t));let f=l(e.l10n).locales;if(f===void 0)return s;let p=new Set(f.map(e=>e.code)),h=[...p].filter(e=>e!==t.defaultLocale&&!t.isLocale(e));h.length>0&&s.push({kind:`locale-without-dir`,locales:h});let g=i.filter(e=>!p.has(e));return g.length>0&&s.push({kind:`dir-not-in-config`,dirs:g}),s}function m(t,n){let r=/* @__PURE__ */ new Map;for(let i of[...t].sort(e)){let e=h(i,n);e!==void 0&&r.set(e,[...r.get(e)??[],i])}return[...r.entries()].filter(([,e])=>e.length>1).map(([e,t])=>({kind:`duplicate-translations`,locale:e,paths:t}))}function h(e,i){let a=e.toLowerCase();if(!r.test(a))return;if(n.some(e=>a===e))return i.defaultLocale;let o=RegExp(`^${t}/([^/]+)/(translations\\.ya?ml)$`,`i`).exec(e);if(o!==null)return i.isLocale(o[1])?o[1]:void 0}function g(e){return e.map(e=>u({code:`configuration/l10n-layout`,message:_(e),data:{...e}}))}function _(e){switch(e.kind){case`default-locale-dir`:return`Ignored \`${t}/${e.dir}/\`: the default locale lives at the project root`;case`stray-file`:return`Ignored files directly under \`${t}/\`, expected \`${t}/<locale>/<path>\`: ${e.paths.join(`, `)}`;case`dir-slug-collision`:return`Two \`${t}\` directories map to the same URL segment: ${e.codes.join(`, `)}`;case`locale-without-dir`:return`Missing \`${t}/<locale>/\` directory for configured locales: ${e.locales.join(`, `)}`;case`dir-not-in-config`:return`\`${t}/\` directories not listed in \`l10n.locales\` are hidden from the language picker: ${e.dirs.join(`, `)}`;case`duplicate-translations`:return`Loaded only the first translations file for \`${e.locale}\`: ${e.paths.join(`, `)}`}}export{p as collectL10nWarnings,g as l10nLayoutProblems,h as translationsFileLocale,d as validateL10nConfig};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{RBAC_TEAM_ANONYMOUS as e,RBAC_TEAM_AUTHENTICATED as t}from"../../
|
|
1
|
+
import{RBAC_TEAM_ANONYMOUS as e,RBAC_TEAM_AUTHENTICATED as t}from"../../shared/rbac/types.js";function n(n){if(n===null)return[e,t];if(n===void 0)return[];let r=n[`*`],i=r===void 0?{...n}:{[e]:r,[t]:r,...n},a=/* @__PURE__ */ new Set;for(let[e,t]of Object.entries(i))e!==`*`&&t!==`none`&&a.add(e.toLowerCase());return[...a].sort()}function r(e,t){let r=n(e);if(t===void 0)return r;let i=n(t),a=(e,t)=>e.includes(t)||e.includes(`authenticated`),o=/* @__PURE__ */ new Set;for(let e of[...r,...i])e===`anonymous`||e===`authenticated`?r.includes(e)&&i.includes(e)&&o.add(e):a(r,e)&&a(i,e)&&o.add(e);return[...o].sort()}export{n as toRbacTeams,r as toRbacTeamsWithinRoute};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{formatCount as e}from"../../shared/log-format.js";import{createLogger as t,stopwatch as n}from"../../shared/logger.js";import{L10N_DIR as r}from"../../shared/constants/l10n/defaults.js";import{collectL10nDirLocales as i,l10nDirOf as a,resolveLocaleContext as o}from"../../shared/utils/l10n.js";import{loadEffectiveConfigHash as s,writeSiteConfig as c}from"../database/site-config.js";import{isConfigChanged as l,isNestedConfigPath as u,loadContentLayerConfig as ee,readNestedConfigProblems as te}from"../config.js";import{contentIgnoreMatcherOf as ne,normalizeHintPaths as re,scanContentFiles as d}from"./scan.js";import{resolveContentPluginsForFile as f}from"../plugins/registry.js";import{resolveContentLayerContext as ie}from"../context.js";import{openContentLayerDatabase as p}from"../database/connection.js";import{garbageCollectOrphanBlobs as ae}from"../database/blobs.js";import"../../shared/constants/app-data.js";import{isPlanFeatureGranted as m}from"../../shared/plan-gates.js";import{listConfiguredProducts as oe,productFolderProblems as se}from"../products.js";import{appDataSystemEntityDescriptor as ce}from"../plugins/internal/app-data/index.js";import{warnWhenProjectPluginsAreNotLoaded as le}from"../plugins/project/loaded-set.js";import{apiOutputFiles as ue,readApiOutputs as de,withApiOutputHints as fe}from"../api-outputs.js";import{createContentPlugins as h}from"../plugins/create-content-plugins.js";import"../plugins/index.js";import{assembleTemplateSet as pe}from"../templates/template-set.js";import{assembleServerModuleSet as me}from"../server-modules/module-set.js";import{listRegisteredAssets as he}from"../database/assets.js";import{createProblem as g}from"../problems/codes.js";import{computeDirtyConfigOrigins as ge,computeDirtyFileOrigins as _e,garbageCollectOrphanedEntities as ve,loadOriginDependencyRows as ye,propagateConfigDependencyInvalidation as be,propagateEntityInvalidation as xe,propagateRetryableProblemInvalidation as Se}from"../database/invalidation.js";import{hasGlobalProblemsForSource as _,listGlobalProblems as Ce,listProblems as we,problemRow as v,replaceGlobalProblemsForKeysTx as Te,replaceGlobalProblemsForSourceTx as y}from"../database/problems.js";import{persistContentFileState as Ee,refreshFileLastModified as De}from"../database/files.js";import{loadContentLayerState as b,partitionIgnoredFiles as Oe}from"../context-state.js";import{garbageCollectOrphanAssetFiles as ke,restoreMissingAssetFiles as Ae}from"../materialization/asset-reconcile.js";import{loadPluginHookState as je}from"../database/plugin-hook-state.js";import{loadRouteQueryDependencyRows as Me}from"../database/route-query-deps.js";import{runSearchSync as Ne}from"./search-sync.js";import{hashContentFiles as x}from"./hash.js";import{selectCurrentContentFiles as S}from"./current-files.js";import{collectL10nWarnings as Pe,l10nLayoutProblems as Fe,validateL10nConfig as Ie}from"./l10n-warnings.js";import{collectAppDataFragmentOwnerIds as Le,reconcilePluginScopedRows as Re,registerSystemEntities as ze}from"../database/finalize.js";import{hasStaleManifestVersion as Be,resetContentDatabase as Ve}from"../database/reset.js";import{readGitLastModified as He}from"./git-last-modified.js";import{createContentScanPlan as Ue}from"./scan-plan.js";import{collectPageRedirects as C}from"../page-redirects.js";import{syncContentFiles as We}from"./sync.js";import{checkAllowedPackages as Ge}from"../allowed-packages.js";import{syncConfigLinkRefs as Ke}from"../link-checker/config-links.js";import{collectCollisionPeerOrigins as qe}from"./slugger/collision-peers.js";async function w(r){let a=ie({contentDir:r.contentDir,context:r.context,currentFileIndex:r.currentFileIndex,logger:r.logger??t(),plugins:h(r.contentDir)}),d=a.contentDir,f=a.logger.child(`content-layer`),m=f.child(`content-layer:scan`),x=f.child(`content-layer:hash`),S=f.child(`content-layer:db`),C=f.child(`content-layer:plan`),w=f.child(`content-layer:diff`),T=f.child(`content-layer:sync`),E=n();f.debug(`Starting content ingest`,{contentDir:d,hints:r.hints??[]});let D=r.database===void 0,O=r.database??await p({contentDir:d,databaseUrl:r.databaseUrl,migrationsDir:r.migrationsDir,runMigrations:r.runMigrations,platform:a.platform,logger:S});try{if(await Be(O)){S.warn(`manifest version changed — resetting content database`);let e=n();await Ve(O,{logger:S}),S.info(`Reset content database`,{durationMs:e()}),a.state=void 0}let t=re(d,r.hints??[]),ie=n(),p=n(),h=await b({context:a,database:O}),D=[...h.filesByPath.values(),...h.ignoredFilesByPath.values()].sort((e,t)=>e.path.localeCompare(t.path));S.debug(`Loaded previous content state`,{loadedCount:D.length,totalCount:D.length,durationMs:p()});let j=await ee({context:a,force:!0});await le({contentDir:d,config:j.config,fs:a.platform.fs,logger:f});let M=l(A(j.config,h.routeBySlug.values()),await s(O)),N=Ie(a.config.l10n);for(let e of N)f.debug(e.message);(M||N.length>0||await _(O,`l10n-config`))&&await O.db.transaction(e=>y(e,`l10n-config`,N.map(e=>v(e,{scope:`global`,source:`l10n-config`})))),(M||j.problems.length>0||await _(O,`config`))&&await O.db.transaction(e=>y(e,`config`,j.problems.map(e=>v(g(e),{scope:`global`,source:`config`}))));let Xe=await Ge(j.config,d,a.platform.fs);(Xe.length>0||await _(O,`allowed-packages`))&&await O.db.transaction(e=>y(e,`allowed-packages`,Xe.map(e=>v(g(e),{scope:`global`,source:`allowed-packages`}))));let P=ne(a.config);Oe(h,P);let F=e=>de({config:a.config,contentDir:d,fs:a.platform.fs,nestedConfigPaths:e,isIgnored:P}),I=M||t.some(u),L=await F([...h.filesByPath.keys()].filter(u)),Ze=n(),R=await Ue({contentDir:d,fs:a.platform.fs,previousFiles:D,hints:fe(t,L),ignoredDirectories:r.ignoredDirectories,configChanged:I,logger:C});C.debug(`Resolved ingest scan plan`,{hints:R.hintedPaths.length,configChanged:I,scanTargets:R.scanTargets.length,affectedPreviousFiles:R.affectedPreviousFiles.length,durationMs:Ze()});let z=new Map(D.map(e=>[e.path,e.hash])),B=await Ye({contentDir:d,currentFileIndex:a.currentFileIndex,fs:a.platform.fs,scanPlan:R,ignoredDirectories:r.ignoredDirectories,scanLogger:m,hashLogger:x,previous:a.currentFileIndex?void 0:z,onProgress:r.onProgress});(R.hintedPaths.length===0||R.hintedPaths.includes(``))&&(L=await F(B.map(e=>e.path).filter(e=>u(e)&&!P(e))));let Qe=await ue({outputs:L,files:B,contentDir:d,fs:a.platform.fs,previous:z}),V=[...B.filter(e=>!L.has(e.path)),...Qe],H=new Map(await He(d,m));for(let[e,t]of L){let n=H.get(t);n!==void 0&&H.set(e,n)}f.info(`Scanned ${e(V.length)} files`,{durationMs:ie()});let $e=n(),U=k(V,R.affectedPreviousFiles,{hintScopes:R.hintScopes,totalPrevious:D.length});w.debug(`Computed content diff`,{...U.summary,durationMs:$e()}),f.info(`Compared with the previous state: ${U.summary.added} added, ${U.summary.changed} changed, ${U.summary.removed} removed, ${U.summary.unchanged} unchanged`);let et=n(),tt=n(),nt=await Ee(O,{upserts:[...U.added.map(e=>({path:e.path,hash:e.hash,lastModifiedAt:H.get(e.path)??null,manifest:JSON.stringify(e.manifest??{}),manifestVersion:e.manifestVersion})),...U.changed.map(e=>({path:e.path,hash:e.nextHash,lastModifiedAt:H.get(e.path)??null,manifest:JSON.stringify(e.file.manifest??{}),manifestVersion:e.file.manifestVersion}))],removedPaths:U.removed.map(e=>e.path),logger:S});S.debug(`Persisted content file state`,{upserts:U.added.length+U.changed.length,removed:U.removed.length,durationMs:tt()});for(let e of U.removed)h.filesByPath.delete(e.path),h.ignoredFilesByPath.delete(e.path);for(let e of nt)(P(e.path)?h.ignoredFilesByPath:h.filesByPath).set(e.path,e);let W=o(a.config.l10n,i(h.filesByPath.keys())),G=a.currentFileIndex!==void 0||I||R.hintedPaths.length===0||R.hintedPaths.includes(``);if(G){let e=Fe(Pe({l10n:a.config.l10n,localeContext:W,filePaths:h.filesByPath.keys()})),t=se(oe(a.config),h.filesByPath.keys());for(let t of e)f.debug(t.message);for(let e of t)f.debug(e.message);await O.db.transaction(async n=>{await y(n,`l10n-layout`,e.map(e=>v(e,{scope:`global`,source:`l10n-layout`}))),await y(n,`products`,t.map(e=>v(g(e),{scope:`global`,source:`products`})))})}let K=G?[...h.filesByPath.keys()].filter(u):[...U.added,...U.changed].map(e=>e.path).filter(e=>u(e)&&!P(e)),q=U.removed.map(e=>e.path).filter(u);if(G||K.length>0||q.length>0){let e=(await te({contentDir:d,fs:a.platform.fs,paths:K})).map(e=>v(g(e),{scope:`global`,source:`nested-config`,key:e.location?.filePath}));await O.db.transaction(t=>G?y(t,`nested-config`,e):Te(t,`nested-config`,[...K,...q],e))}let J=V.flatMap(e=>{let t=h.filesByPath.get(e.path),n=H.get(e.path)??null;return!t||t.lastModifiedAt===n?[]:(h.filesByPath.set(e.path,{...t,lastModifiedAt:n}),[{path:e.path,lastModifiedAt:n}])});await De(O,J,S),await xe(O,{changedFilePaths:[...U.added.map(e=>e.path),...U.changed.map(e=>e.path),...U.removed.map(e=>e.path),...J.map(e=>e.path)],membershipChangedPaths:[...U.added,...U.removed].map(e=>e.path).filter(e=>!P(e)),logger:S});let rt=a.pluginRegistry.listExtractAllPlugins(),it=a.pluginRegistry.listFinalizePlugins(),at=new Set(it.map(e=>e.id)),Y=[...Le(h.entitiesByKey.get(`app-data`)?.loaderArgs??null)].filter(e=>!at.has(e)),ot=await Re(O,[...rt.map(e=>({pluginId:e.id,hook:`extractAll`})),...it.map(e=>({pluginId:e.id,hook:`finalize`}))],{logger:S});Y.length>0&&await ze(O,[ce({fragmentUpdates:new Map(Y.map(e=>[e,null]))})],{logger:S});let[X,st,ct]=await Promise.all([ye(O),Me(O),je(O)]),lt=ge(X,a.config),ut=_e(X,h);await be(O,{config:a.config,logger:S}),await Se(O,{logger:S}),await ve(O,S),(ot||Y.length>0)&&(a.state=void 0,h=await b({context:a,database:O}));let dt=await Ce(O,`slugger`),ft=qe(dt,{touchedPaths:[...U.added.map(e=>e.path),...U.changed.map(e=>e.path),...U.removed.map(e=>e.path),...lt.map(e=>e.originFilePath),...ut.map(e=>e.originFilePath)],storedFilesByPath:h.filesByPath}),Z=Je({context:a,diff:U,currentFiles:V,dirtyOrigins:[...lt,...ut,...ft],hintedPaths:R.hintedPaths,storedFilesByPath:h.filesByPath,storedPluginsByPath:h.filePluginsByPath}),pt=n(),Q=await We({context:a,database:O,localeContext:W,files:Z,removedPaths:new Set(U.removed.map(e=>e.path)),pluginScopedOriginRows:X,routeQueryRows:st,hookState:ct,storedSluggerProblems:dt,captureRevisions:r.captureRevisions,onFileSynced:(e,t)=>r.onProgress?.({stage:`sync`,done:e,total:t})});T.debug(`Synchronized content routes`,{files:Z.length,durationMs:pt()}),await Ke({database:O,config:a.config,configChanged:M,logger:S}),f.info(`Updated content state: ${U.added.length+U.changed.length} upserted, ${U.removed.length} removed, ${Z.length} synced`,{durationMs:et()}),r.searchSync!==!1&&await Ne({context:a,database:O,inlineSearchables:Q.inlineSearchables});let mt=await ae(O,{logger:S});mt.removed>0&&S.debug(`blob GC removed orphan blobs`,{count:mt.removed});let $=await ke(a.pluginIO,{contentDir:d,database:O,logger:S});if($.removedRows>0&&S.debug(`asset GC removed orphan assets`,{count:$.removedRows,swept:$.sweptFilenames.length}),r.reconcileAssetFiles??!0){let e=await Ae(a.pluginIO,{contentDir:d,registered:await he(O)});e.restored.length>0&&f.info(`Restored ${e.restored.length} missing content ${e.restored.length===1?`asset`:`assets`}`),e.missingSources.length>0&&S.warn(`registered assets are missing their source file`,{paths:e.missingSources}),e.failed.length>0&&S.warn(`could not restore missing content assets; /content-assets/* will 404`,{paths:e.failed})}let{routeBySlug:ht}=await b({context:a,database:O});a.config=A(j.config,ht.values()),await c(O,A(j.siteConfig,ht.values()),a.config);try{await O.db.transaction(e=>y(e,`system`,[]))}catch(e){S.warn(`could not clear the stored run failure`,{error:e instanceof Error?e.message:String(e)})}let gt=r.collectProblems===!1?[]:await we(O);f.debug(`Content ingest complete`,{durationMs:E()});let _t=a.state??h,vt=pe({registry:a.pluginRegistry,routes:_t.routeBySlug.values()}),yt=me({registry:a.pluginRegistry,entities:_t.entitiesByKey.values()});return{contentDir:d,diff:U,currentFiles:B,previousFiles:R.affectedPreviousFiles,problems:gt,templateSet:vt,serverModuleSet:yt,inlineSearchables:Q.inlineSearchables,routeChanges:{...Q.routeChanges,unbounded:Q.routeChanges.unbounded||M}}}finally{D&&await O.close()}}function Je(e){let t=/* @__PURE__ */ new Map;for(let n of e.diff.added)t.set(n.path,n);for(let n of e.diff.changed)t.set(n.path,n.file);let n=new Map(e.currentFiles.map(e=>[e.path,e]));for(let n of e.currentFiles)t.has(n.path)||E(e.context,n,e.storedPluginsByPath.get(n.path))&&t.set(n.path,n);for(let r of e.dirtyOrigins){let i=n.get(r.originFilePath)??O(e.context.contentDir,e.storedFilesByPath.get(r.originFilePath));i&&t.set(i.path,i)}let r=[...t.keys(),...e.diff.removed.map(e=>e.path)];for(let i of T(r,e.storedFilesByPath.keys())){if(t.has(i))continue;let r=n.get(i)??O(e.context.contentDir,e.storedFilesByPath.get(i));r&&t.set(i,r)}return Array.from(t.values())}function T(e,t){let n=new Set([...e].filter(e=>!e.startsWith(`${r}/`))),i=[];for(let e of t){let t=a(e);t!==void 0&&n.has(e.slice(`@l10n/${t}/`.length))&&i.push(e)}return i}function E(e,t,n){let r=f(e,t.path).map(e=>({id:e.id,version:e.version})).sort((e,t)=>e.id.localeCompare(t.id)),i=[...n??[]].sort((e,t)=>e.pluginId.localeCompare(t.pluginId));return r.length!==i.length||r.some((e,t)=>{let n=i[t];return n===void 0||n.pluginId!==e.id||n.pluginVersion!==e.version})}function D(e,t){return`${e.replace(/\/$/,``)}/${t}`}function O(e,t){if(t)return{absolutePath:D(e,t.path),hash:t.hash,manifestVersion:t.manifestVersion,path:t.path}}function k(e,t,n={}){let r=new Map(e.map(e=>[e.path,e])),i=new Map(t.map(e=>[e.path,e])),a=n.hintScopes??[],o=[],s=[],c=[],l=[];for(let t of e){let e=i.get(t.path);if(!e){o.push(t);continue}if(e.hash!==t.hash){s.push({path:t.path,previousHash:e.hash,nextHash:t.hash,file:t});continue}c.push(t)}for(let e of t)r.has(e.path)||l.push(e);return{added:o,changed:s,removed:l,unchanged:c,hintScopes:a,summary:{added:o.length,changed:s.length,removed:l.length,unchanged:c.length,totalCurrent:n.totalPrevious===void 0?e.length:n.totalPrevious-l.length+o.length,totalPrevious:n.totalPrevious??t.length}}}async function Ye(e){if(e.currentFileIndex){let t=S({currentFileIndex:e.currentFileIndex,hintScopes:e.scanPlan.hintScopes,hintedPaths:e.scanPlan.hintedPaths});return e.scanLogger.debug(`Using provided current file snapshot`,{count:t.length,totalCount:e.currentFileIndex.size}),t}let t=n(),r=await d({fs:e.fs,rootDir:e.contentDir,targets:e.scanPlan.scanTargets,ignoredDirectories:e.ignoredDirectories,logger:e.scanLogger});e.scanLogger.debug(`Scanned content files`,{count:r.length,durationMs:t()});let i=n(),a=await x(r,{fs:e.fs,logger:e.hashLogger,previous:e.previous,onFileHashed:(t,n)=>e.onProgress?.({stage:`scan`,done:t,total:n})});return e.hashLogger.debug(`Computed content hashes`,{count:a.length,durationMs:i()}),a}function A(e,t){let n=m(e,`redirects`)?C(t):{};return{...e,redirects:{...n,...e.redirects}}}export{k as diffContentFiles,w as runContentIngest};
|
|
1
|
+
import{isPlanFeatureGranted as e}from"../../shared/plan-gates.js";import{assembleTemplateSet as t}from"../templates/template-set.js";import{assembleServerModuleSet as n}from"../server-modules/module-set.js";import{formatCount as r}from"../../shared/log-format.js";import{createLogger as i,stopwatch as a}from"../../shared/logger.js";import{L10N_DIR as o}from"../../shared/constants/l10n/defaults.js";import{collectL10nDirLocales as s,l10nDirOf as c,resolveLocaleContext as l}from"../../shared/utils/l10n.js";import{loadEffectiveConfigHash as ee,writeSiteConfig as te}from"../database/site-config.js";import{isConfigChanged as ne,isNestedConfigPath as u,loadContentLayerConfig as re,readNestedConfigProblems as ie}from"../config.js";import{contentIgnoreMatcherOf as ae,normalizeHintPaths as oe,scanContentFiles as d}from"./scan.js";import{resolveContentPluginsForFile as f}from"../plugins/registry.js";import{resolveContentLayerContext as p}from"../context.js";import{openContentLayerDatabase as m}from"../database/connection.js";import{garbageCollectOrphanBlobs as se}from"../database/blobs.js";import"../../shared/constants/app-data.js";import{listConfiguredProducts as ce,productFolderProblems as le}from"../products.js";import{appDataSystemEntityDescriptor as ue}from"../plugins/internal/app-data/index.js";import{warnWhenProjectPluginsAreNotLoaded as de}from"../plugins/project/loaded-set.js";import{apiOutputFiles as fe,readApiOutputs as pe,withApiOutputHints as me}from"../api-outputs.js";import{createContentPlugins as h}from"../plugins/create-content-plugins.js";import"../plugins/index.js";import{listRegisteredAssets as he}from"../database/assets.js";import{createProblem as g}from"../problems/codes.js";import{computeDirtyConfigOrigins as ge,computeDirtyFileOrigins as _e,garbageCollectOrphanedEntities as ve,loadOriginDependencyRows as ye,propagateConfigDependencyInvalidation as be,propagateEntityInvalidation as xe,propagateRetryableProblemInvalidation as Se}from"../database/invalidation.js";import{hasGlobalProblemsForSource as _,listGlobalProblems as Ce,listProblems as we,problemRow as v,replaceGlobalProblemsForKeysTx as Te,replaceGlobalProblemsForSourceTx as y}from"../database/problems.js";import{persistContentFileState as Ee,refreshFileLastModified as De}from"../database/files.js";import{loadContentLayerState as b,partitionIgnoredFiles as Oe,reloadContentLayerEntities as ke}from"../context-state.js";import{garbageCollectOrphanAssetFiles as Ae,restoreMissingAssetFiles as je}from"../materialization/asset-reconcile.js";import{resolveRevisionCapture as Me}from"../revisions/config.js";import{loadPluginHookState as Ne}from"../database/plugin-hook-state.js";import{loadRouteQueryDependencyRows as Pe}from"../database/route-query-deps.js";import{createEntityMaterializer as Fe}from"../materialization/materializer.js";import{runSearchSync as Ie}from"./search-sync.js";import{hashContentFiles as x}from"./hash.js";import{selectCurrentContentFiles as S}from"./current-files.js";import{collectL10nWarnings as Le,l10nLayoutProblems as Re,validateL10nConfig as ze}from"./l10n-warnings.js";import{collectAppDataFragmentOwnerIds as Be,reconcilePluginScopedRows as Ve,registerSystemEntities as He}from"../database/finalize.js";import{hasStaleManifestVersion as Ue,resetContentDatabase as We}from"../database/reset.js";import{readGitLastModified as Ge}from"./git-last-modified.js";import{createContentScanPlan as Ke}from"./scan-plan.js";import{collectPageRedirects as C}from"../page-redirects.js";import{syncContentFiles as qe}from"./sync.js";import{checkAllowedPackages as Je}from"../allowed-packages.js";import{syncConfigLinkRefs as Ye}from"../link-checker/config-links.js";import{collectCollisionPeerOrigins as Xe}from"./slugger/collision-peers.js";async function w(e){let o=p({contentDir:e.contentDir,context:e.context,currentFileIndex:e.currentFileIndex,logger:e.logger??i(),plugins:h(e.contentDir)}),c=o.contentDir,d=o.logger.child(`content-layer`),f=d.child(`content-layer:scan`),x=d.child(`content-layer:hash`),S=d.child(`content-layer:db`),C=d.child(`content-layer:plan`),w=d.child(`content-layer:diff`),T=d.child(`content-layer:sync`),E=e.captureRevisions===!0,D=a();d.debug(`Starting content ingest`,{contentDir:c,hints:e.hints??[]});let O=e.database===void 0,j=e.database??await m({contentDir:c,databaseUrl:e.databaseUrl,migrationsDir:e.migrationsDir,runMigrations:e.runMigrations,platform:o.platform,logger:S});try{if(await Ue(j)){S.warn(`manifest version changed — resetting content database`);let e=a();await We(j,{logger:S}),S.info(`Reset content database`,{durationMs:e()}),o.state=void 0}let i=oe(c,e.hints??[]),p=a(),m=a(),h=await b({context:o,database:j}),O=[...h.filesByPath.values(),...h.ignoredFilesByPath.values()].sort((e,t)=>e.path.localeCompare(t.path));S.debug(`Loaded previous content state`,{loadedCount:O.length,totalCount:O.length,durationMs:m()});let M=await re({context:o,force:!0});await de({contentDir:c,config:M.config,fs:o.platform.fs,logger:d});let N=ne(A(M.config,h.routeBySlug.values()),await ee(j)),P=ze(o.config.l10n);for(let e of P)d.debug(e.message);(N||P.length>0||await _(j,`l10n-config`))&&await j.db.transaction(e=>y(e,`l10n-config`,P.map(e=>v(e,{scope:`global`,source:`l10n-config`})))),(N||M.problems.length>0||await _(j,`config`))&&await j.db.transaction(e=>y(e,`config`,M.problems.map(e=>v(g(e),{scope:`global`,source:`config`}))));let F=await Je(M.config,c,o.platform.fs);(F.length>0||await _(j,`allowed-packages`))&&await j.db.transaction(e=>y(e,`allowed-packages`,F.map(e=>v(g(e),{scope:`global`,source:`allowed-packages`}))));let I=ae(o.config);Oe(h,I);let L=e=>pe({config:o.config,contentDir:c,fs:o.platform.fs,nestedConfigPaths:e,isIgnored:I}),R=N||i.some(u),z=await L([...h.filesByPath.keys()].filter(u)),$e=a(),B=await Ke({contentDir:c,fs:o.platform.fs,previousFiles:O,hints:me(i,z),ignoredDirectories:e.ignoredDirectories,configChanged:R,logger:C});C.debug(`Resolved ingest scan plan`,{hints:B.hintedPaths.length,configChanged:R,scanTargets:B.scanTargets.length,affectedPreviousFiles:B.affectedPreviousFiles.length,durationMs:$e()});let et=new Map(O.map(e=>[e.path,e.hash])),V=await Qe({contentDir:c,currentFileIndex:o.currentFileIndex,fs:o.platform.fs,scanPlan:B,ignoredDirectories:e.ignoredDirectories,scanLogger:f,hashLogger:x,previous:o.currentFileIndex?void 0:et,onProgress:e.onProgress});(B.hintedPaths.length===0||B.hintedPaths.includes(``))&&(z=await L(V.map(e=>e.path).filter(e=>u(e)&&!I(e))));let tt=await fe({outputs:z,files:V,contentDir:c,fs:o.platform.fs,previous:et}),H=[...V.filter(e=>!z.has(e.path)),...tt],U=new Map(await Ge(c,f));for(let[e,t]of z){let n=U.get(t);n!==void 0&&U.set(e,n)}d.info(`Scanned ${r(H.length)} files`,{durationMs:p()});let nt=a(),W=k(H,B.affectedPreviousFiles,{hintScopes:B.hintScopes,totalPrevious:O.length});w.debug(`Computed content diff`,{...W.summary,durationMs:nt()}),d.info(`Compared with the previous state: ${W.summary.added} added, ${W.summary.changed} changed, ${W.summary.removed} removed, ${W.summary.unchanged} unchanged`);let rt=a(),it=a(),at=await Ee(j,{upserts:[...W.added.map(e=>({path:e.path,hash:e.hash,lastModifiedAt:U.get(e.path)??null,manifest:JSON.stringify(e.manifest??{}),manifestVersion:e.manifestVersion})),...W.changed.map(e=>({path:e.path,hash:e.nextHash,lastModifiedAt:U.get(e.path)??null,manifest:JSON.stringify(e.file.manifest??{}),manifestVersion:e.file.manifestVersion}))],removedPaths:W.removed.map(e=>e.path),logger:S});S.debug(`Persisted content file state`,{upserts:W.added.length+W.changed.length,removed:W.removed.length,durationMs:it()});for(let e of W.removed)h.filesByPath.delete(e.path),h.ignoredFilesByPath.delete(e.path);for(let e of at)(I(e.path)?h.ignoredFilesByPath:h.filesByPath).set(e.path,e);let ot=l(o.config.l10n,s(h.filesByPath.keys())),G=o.currentFileIndex!==void 0||R||B.hintedPaths.length===0||B.hintedPaths.includes(``);if(G){let e=Re(Le({l10n:o.config.l10n,localeContext:ot,filePaths:h.filesByPath.keys()})),t=le(ce(o.config),h.filesByPath.keys());for(let t of e)d.debug(t.message);for(let e of t)d.debug(e.message);await j.db.transaction(async n=>{await y(n,`l10n-layout`,e.map(e=>v(e,{scope:`global`,source:`l10n-layout`}))),await y(n,`products`,t.map(e=>v(g(e),{scope:`global`,source:`products`})))})}let K=G?[...h.filesByPath.keys()].filter(u):[...W.added,...W.changed].map(e=>e.path).filter(e=>u(e)&&!I(e)),st=W.removed.map(e=>e.path).filter(u);if(G||K.length>0||st.length>0){let e=(await ie({contentDir:c,fs:o.platform.fs,paths:K})).map(e=>v(g(e),{scope:`global`,source:`nested-config`,key:e.location?.filePath}));await j.db.transaction(t=>G?y(t,`nested-config`,e):Te(t,`nested-config`,[...K,...st],e))}let ct=H.flatMap(e=>{let t=h.filesByPath.get(e.path),n=U.get(e.path)??null;return!t||t.lastModifiedAt===n?[]:(h.filesByPath.set(e.path,{...t,lastModifiedAt:n}),[{path:e.path,lastModifiedAt:n}])});await De(j,ct,S);let lt=await xe(j,{changedFilePaths:[...W.added.map(e=>e.path),...W.changed.map(e=>e.path),...W.removed.map(e=>e.path),...ct.map(e=>e.path)],membershipChangedPaths:[...W.added,...W.removed].map(e=>e.path).filter(e=>!I(e)),logger:S}),ut=o.pluginRegistry.listExtractAllPlugins(),dt=o.pluginRegistry.listFinalizePlugins(),ft=new Set(dt.map(e=>e.id)),q=[...Be(h.entitiesByKey.get(`app-data`)?.loaderArgs??null)].filter(e=>!ft.has(e)),pt=await Ve(j,[...ut.map(e=>({pluginId:e.id,hook:`extractAll`})),...dt.map(e=>({pluginId:e.id,hook:`finalize`}))],{logger:S});q.length>0&&await He(j,[ue({fragmentUpdates:new Map(q.map(e=>[e,null]))})],{logger:S});let[J,mt,ht]=await Promise.all([ye(j),Pe(j),Ne(j)]),gt=ge(J,o.config),_t=_e(J,h),vt=await be(j,{config:o.config,logger:S}),yt=await Se(j,{logger:S});if((await ve(j,S)).removedOrphans)await ke({state:h,database:j});else for(let e of[...lt.invalidatedKeys,...vt.invalidatedKeys,...yt.invalidatedKeys]){let t=h.entitiesByKey.get(e);t&&(t.status=`stale`)}(pt||q.length>0)&&(o.state=void 0,h=await b({context:o,database:j}));let bt=await Ce(j,`slugger`),xt=Xe(bt,{touchedPaths:[...W.added.map(e=>e.path),...W.changed.map(e=>e.path),...W.removed.map(e=>e.path),...gt.map(e=>e.originFilePath),..._t.map(e=>e.originFilePath)],storedFilesByPath:h.filesByPath}),Y=Ze({context:o,diff:W,currentFiles:H,dirtyOrigins:[...gt,..._t,...xt],hintedPaths:B.hintedPaths,storedFilesByPath:h.filesByPath,storedPluginsByPath:h.filePluginsByPath}),St=a(),X=Me(o.config,E),Z=await qe({context:o,database:j,localeContext:ot,files:Y,removedPaths:new Set(W.removed.map(e=>e.path)),pluginScopedOriginRows:J,routeQueryRows:mt,hookState:ht,storedSluggerProblems:bt,revisions:X,onFileSynced:(t,n)=>e.onProgress?.({stage:`sync`,done:t,total:n})});o.state=Z.state,T.debug(`Synchronized content routes`,{files:Y.length,durationMs:St()}),await Ye({database:j,config:o.config,configChanged:N,logger:S}),d.info(`Updated content state: ${W.added.length+W.changed.length} upserted, ${W.removed.length} removed, ${Y.length} synced`,{durationMs:rt()}),e.searchSync!==!1&&await Ie({context:o,database:j,inlineSearchables:Z.inlineSearchables,materializer:Fe({deps:{...o,state:Z.state},database:j,revisions:X})});let Ct=await se(j,{logger:S});Ct.removed>0&&S.debug(`blob GC removed orphan blobs`,{count:Ct.removed});let Q=await Ae(o.pluginIO,{contentDir:c,database:j,logger:S});if(Q.removedRows>0&&S.debug(`asset GC removed orphan assets`,{count:Q.removedRows,swept:Q.sweptFilenames.length}),e.reconcileAssetFiles??!0){let e=await je(o.pluginIO,{contentDir:c,registered:await he(j)});e.restored.length>0&&d.info(`Restored ${e.restored.length} missing content ${e.restored.length===1?`asset`:`assets`}`),e.missingSources.length>0&&S.warn(`registered assets are missing their source file`,{paths:e.missingSources}),e.failed.length>0&&S.warn(`could not restore missing content assets; /content-assets/* will 404`,{paths:e.failed})}let{routeBySlug:wt}=Z.state;o.config=A(M.config,wt.values()),await te(j,A(M.siteConfig,wt.values()),o.config);try{await j.db.transaction(e=>y(e,`system`,[]))}catch(e){S.warn(`could not clear the stored run failure`,{error:e instanceof Error?e.message:String(e)})}let Tt=e.collectProblems===!1?[]:await we(j);d.debug(`Content ingest complete`,{durationMs:D()});let $=Z.state,Et=t({registry:o.pluginRegistry,routes:$.routeBySlug.values()}),Dt=n({registry:o.pluginRegistry,entities:$.entitiesByKey.values()});return{contentDir:c,diff:W,currentFiles:V,previousFiles:B.affectedPreviousFiles,problems:Tt,templateSet:Et,serverModuleSet:Dt,inlineSearchables:Z.inlineSearchables,routeChanges:{...Z.routeChanges,unbounded:Z.routeChanges.unbounded||N}}}finally{O&&await j.close()}}function Ze(e){let t=/* @__PURE__ */ new Map;for(let n of e.diff.added)t.set(n.path,n);for(let n of e.diff.changed)t.set(n.path,n.file);let n=new Map(e.currentFiles.map(e=>[e.path,e]));for(let n of e.currentFiles)t.has(n.path)||E(e.context,n,e.storedPluginsByPath.get(n.path))&&t.set(n.path,n);for(let r of e.dirtyOrigins){let i=n.get(r.originFilePath)??O(e.context.contentDir,e.storedFilesByPath.get(r.originFilePath));i&&t.set(i.path,i)}let r=[...t.keys(),...e.diff.removed.map(e=>e.path)];for(let i of T(r,e.storedFilesByPath.keys())){if(t.has(i))continue;let r=n.get(i)??O(e.context.contentDir,e.storedFilesByPath.get(i));r&&t.set(i,r)}return Array.from(t.values())}function T(e,t){let n=new Set([...e].filter(e=>!e.startsWith(`${o}/`))),r=[];for(let e of t){let t=c(e);t!==void 0&&n.has(e.slice(`@l10n/${t}/`.length))&&r.push(e)}return r}function E(e,t,n){let r=f(e,t.path).map(e=>({id:e.id,version:e.version})).sort((e,t)=>e.id.localeCompare(t.id)),i=[...n??[]].sort((e,t)=>e.pluginId.localeCompare(t.pluginId));return r.length!==i.length||r.some((e,t)=>{let n=i[t];return n===void 0||n.pluginId!==e.id||n.pluginVersion!==e.version})}function D(e,t){return`${e.replace(/\/$/,``)}/${t}`}function O(e,t){if(t)return{absolutePath:D(e,t.path),hash:t.hash,manifestVersion:t.manifestVersion,path:t.path}}function k(e,t,n={}){let r=new Map(e.map(e=>[e.path,e])),i=new Map(t.map(e=>[e.path,e])),a=n.hintScopes??[],o=[],s=[],c=[],l=[];for(let t of e){let e=i.get(t.path);if(!e){o.push(t);continue}if(e.hash!==t.hash){s.push({path:t.path,previousHash:e.hash,nextHash:t.hash,file:t});continue}c.push(t)}for(let e of t)r.has(e.path)||l.push(e);return{added:o,changed:s,removed:l,unchanged:c,hintScopes:a,summary:{added:o.length,changed:s.length,removed:l.length,unchanged:c.length,totalCurrent:n.totalPrevious===void 0?e.length:n.totalPrevious-l.length+o.length,totalPrevious:n.totalPrevious??t.length}}}async function Qe(e){if(e.currentFileIndex){let t=S({currentFileIndex:e.currentFileIndex,hintScopes:e.scanPlan.hintScopes,hintedPaths:e.scanPlan.hintedPaths});return e.scanLogger.debug(`Using provided current file snapshot`,{count:t.length,totalCount:e.currentFileIndex.size}),t}let t=a(),n=await d({fs:e.fs,rootDir:e.contentDir,targets:e.scanPlan.scanTargets,ignoredDirectories:e.ignoredDirectories,logger:e.scanLogger});e.scanLogger.debug(`Scanned content files`,{count:n.length,durationMs:t()});let r=a(),i=await x(n,{fs:e.fs,logger:e.hashLogger,previous:e.previous,onFileHashed:(t,n)=>e.onProgress?.({stage:`scan`,done:t,total:n})});return e.hashLogger.debug(`Computed content hashes`,{count:i.length,durationMs:r()}),i}function A(t,n){let r=e(t,`redirects`)?C(n):{};return{...t,redirects:{...r,...t.redirects}}}export{k as diffContentFiles,w as runContentIngest};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { PlatformFs } from '../../platform/types/fs.js';
|
|
2
2
|
import type { Logger } from '../../shared/logger.js';
|
|
3
|
-
export declare const DEFAULT_IGNORED_DIRECTORIES: readonly ['.git', '.realm', '.realm-lazy', '.realm-sahpool', 'dist', 'node_modules'];
|
|
3
|
+
export declare const DEFAULT_IGNORED_DIRECTORIES: readonly ['.git', '.realm', '.realm-lazy', '.realm-sahpool', '@sdk', 'dist', 'node_modules'];
|
|
4
4
|
export type ContentHint = string;
|
|
5
5
|
export type ContentPathScope = {
|
|
6
6
|
path: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{toPosixPath as e}from"../../shared/utils/to-posix-path.js";import{REDOCLY_CONFIG_FILENAMES as t}from"../config.js";import{compileContentGlob as n}from"../plugins/utils/glob.js";import{isAbsolute as r,join as i,posix as a,relative as o,resolve as s}from"node:path";const c=[`.git`,`.realm`,`.realm-lazy`,`.realm-sahpool`,`dist`,`node_modules`];async function l(e){let t=s(e.rootDir),n=new Set(e.ignoredDirectories??c),r=e.targets===void 0?[{path:``,absolutePath:t,recursive:!0}]:d(e.targets),a=[],o=/* @__PURE__ */ new Set,l=e.logger,u=e=>{let r=v(t,e);if(!(r===``||o.has(r))){if(_(r,n)){l?.debug(`Skipping ignored file`,{path:r});return}o.add(r),a.push({path:r,absolutePath:e})}},f=async r=>{let a=await e.fs.readdir(r);for(let e of a){let a=i(r,e.name),o=v(t,a);if(e.type===`symlink`){l?.debug(`Skipping symbolic link`,{path:o});continue}if(e.type===`directory`){if(_(o,n)){l?.debug(`Skipping ignored directory`,{name:e.name,path:o});continue}await f(a);continue}if(e.type!==`file`){l?.debug(`Skipping non-file entry`,{path:o});continue}u(a)}};for(let e of r){if(e.recursive){if(e.path!==``&&_(e.path,n)){l?.debug(`Skipping ignored scan target`,{path:e.path});continue}await f(e.absolutePath);continue}u(e.absolutePath)}return a.sort((e,t)=>e.path.localeCompare(t.path)),a}function u(e,t){let n=/* @__PURE__ */ new Set,r=s(e);for(let e of t){let t=b(r,e);t!==null&&n.add(t)}return[...n].sort((e,t)=>e.localeCompare(t))}function d(e){let t=[];for(let n of[...e].sort(x))t.some(e=>p(e,n.path))||t.push(n);return t}function f(e,t){return t.some(t=>p(t,e))}function p(e,t){return e.path===``?e.recursive:e.path===t||e.recursive&&t.startsWith(`${e.path}/`)}const m=/* @__PURE__ */ new WeakMap;function h(e){let t=m.get(e);return t===void 0&&(t=g(e.ignore),m.set(e,t)),t}function g(e){let r=typeof e==`string`?[e]:Array.isArray(e)?e:[],i=[];for(let e of r){if(typeof e!=`string`)continue;let t=e.startsWith(`!`),r=(t?e.slice(1):e).replace(/^\.?\/+/,``).replace(/(?<!\/)\/+$/,``);r!==``&&i.push({negated:t,matches:n([r,`${r}/**`])})}return e=>{if(t.includes(e))return!1;let n=!1;for(let t of i)t.matches(e)&&(n=!t.negated);return n}}function _(t,n){let r=n instanceof Set?n:new Set(n);return e(t).split(`/`).filter(Boolean).some(e=>r.has(e))}function v(e,t){return w(o(s(e),t))}function y(e,t){if(/^[a-z][a-z\d+.-]*:\/\//i.test(t))return null;let n=v(e,t);return n===``||n===`..`||n.startsWith(`../`)||r(n)||/^[a-z]:/i.test(n)?null:n}function b(e,t){if(t.trim()===``)return null;if(r(t)){let n=o(e,t);return n===``||C(n)?w(n):null}return w(t)}function x(e,t){let n=S(e.path),r=S(t.path);return n===r?e.recursive===t.recursive?e.path.localeCompare(t.path):e.recursive?-1:1:n-r}function S(e){return e===``?0:e.split(`/`).filter(Boolean).length}function C(e){return e!==`..`&&!e.startsWith(`..${a.sep}`)&&!e.startsWith(`..\\`)}function w(t){let n=a.normalize(e(t)).replace(/^\.\//,``);return n===`.`?``:n}export{c as DEFAULT_IGNORED_DIRECTORIES,d as compactScopes,h as contentIgnoreMatcherOf,g as createContentIgnoreMatcher,_ as isIgnoredContentPath,f as isPathCoveredByScopes,u as normalizeHintPaths,l as scanContentFiles,p as scopeCoversPath,y as toContainedContentPath,v as toContentPath};
|
|
1
|
+
import{toPosixPath as e}from"../../shared/utils/to-posix-path.js";import{REDOCLY_CONFIG_FILENAMES as t}from"../config.js";import{compileContentGlob as n}from"../plugins/utils/glob.js";import{isAbsolute as r,join as i,posix as a,relative as o,resolve as s}from"node:path";const c=[`.git`,`.realm`,`.realm-lazy`,`.realm-sahpool`,`@sdk`,`dist`,`node_modules`];async function l(e){let t=s(e.rootDir),n=new Set(e.ignoredDirectories??c),r=e.targets===void 0?[{path:``,absolutePath:t,recursive:!0}]:d(e.targets),a=[],o=/* @__PURE__ */ new Set,l=e.logger,u=e=>{let r=v(t,e);if(!(r===``||o.has(r))){if(_(r,n)){l?.debug(`Skipping ignored file`,{path:r});return}o.add(r),a.push({path:r,absolutePath:e})}},f=async r=>{let a=await e.fs.readdir(r);for(let e of a){let a=i(r,e.name),o=v(t,a);if(e.type===`symlink`){l?.debug(`Skipping symbolic link`,{path:o});continue}if(e.type===`directory`){if(_(o,n)){l?.debug(`Skipping ignored directory`,{name:e.name,path:o});continue}await f(a);continue}if(e.type!==`file`){l?.debug(`Skipping non-file entry`,{path:o});continue}u(a)}};for(let e of r){if(e.recursive){if(e.path!==``&&_(e.path,n)){l?.debug(`Skipping ignored scan target`,{path:e.path});continue}await f(e.absolutePath);continue}u(e.absolutePath)}return a.sort((e,t)=>e.path.localeCompare(t.path)),a}function u(e,t){let n=/* @__PURE__ */ new Set,r=s(e);for(let e of t){let t=b(r,e);t!==null&&n.add(t)}return[...n].sort((e,t)=>e.localeCompare(t))}function d(e){let t=[];for(let n of[...e].sort(x))t.some(e=>p(e,n.path))||t.push(n);return t}function f(e,t){return t.some(t=>p(t,e))}function p(e,t){return e.path===``?e.recursive:e.path===t||e.recursive&&t.startsWith(`${e.path}/`)}const m=/* @__PURE__ */ new WeakMap;function h(e){let t=m.get(e);return t===void 0&&(t=g(e.ignore),m.set(e,t)),t}function g(e){let r=typeof e==`string`?[e]:Array.isArray(e)?e:[],i=[];for(let e of r){if(typeof e!=`string`)continue;let t=e.startsWith(`!`),r=(t?e.slice(1):e).replace(/^\.?\/+/,``).replace(/(?<!\/)\/+$/,``);r!==``&&i.push({negated:t,matches:n([r,`${r}/**`])})}return e=>{if(t.includes(e))return!1;let n=!1;for(let t of i)t.matches(e)&&(n=!t.negated);return n}}function _(t,n){let r=n instanceof Set?n:new Set(n);return e(t).split(`/`).filter(Boolean).some(e=>r.has(e))}function v(e,t){return w(o(s(e),t))}function y(e,t){if(/^[a-z][a-z\d+.-]*:\/\//i.test(t))return null;let n=v(e,t);return n===``||n===`..`||n.startsWith(`../`)||r(n)||/^[a-z]:/i.test(n)?null:n}function b(e,t){if(t.trim()===``)return null;if(r(t)){let n=o(e,t);return n===``||C(n)?w(n):null}return w(t)}function x(e,t){let n=S(e.path),r=S(t.path);return n===r?e.recursive===t.recursive?e.path.localeCompare(t.path):e.recursive?-1:1:n-r}function S(e){return e===``?0:e.split(`/`).filter(Boolean).length}function C(e){return e!==`..`&&!e.startsWith(`..${a.sep}`)&&!e.startsWith(`..\\`)}function w(t){let n=a.normalize(e(t)).replace(/^\.\//,``);return n===`.`?``:n}export{c as DEFAULT_IGNORED_DIRECTORIES,d as compactScopes,h as contentIgnoreMatcherOf,g as createContentIgnoreMatcher,_ as isIgnoredContentPath,f as isPathCoveredByScopes,u as normalizeHintPaths,l as scanContentFiles,p as scopeCoversPath,y as toContainedContentPath,v as toContentPath};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{hashJsonValue as e}from"../database/utils/json.js";import{SEARCH_GROUP_FACET_FIELD as t,resolveSearchFacets as n}from"../../shared/search/facets.js";import{hashContentBytes as r}from"../../shared/utils/content-hash.js";import{formatDuration as i}from"../../shared/log-format.js";import{isRbacScope as a}from"../../server/rbac/engine.js";import{normalizeRouteSlug as o}from"../../shared/utils/routes/normalize-route-slug.js";import{entities as s,pageMarkdown as c,routeQueryDeps as l,searchDocuments as u}from"../database/schema.js";import{computePathPrefixes as d}from"../../shared/utils/routes/compute-path-prefixes.js";import{replaceDataRefMarkers as f}from"../materialization/data-ref.js";import{batchValues as p,normalizeBatchSize as m}from"../database/connection.js";import{resolveDataRef as h}from"../database/blobs.js";import{createRouteQueryEvaluator as g,dedupeRouteQueryDependencies as _,isRouteQueryStale as v}from"../plugins/utils/route-index.js";import{listConfiguredProducts as y}from"../products.js";import{userMarkdocSchemaDigest as b}from"../markdown/constants/tag-sets.js";import{curationKeywordsOf as x}from"../search/keywords.js";import{findProductForSlug as S}from"../../shared/products.js";import{createMarkdownCopyRule as C}from"../markdown-copy-rule.js";import{createProblem as w}from"../problems/codes.js";import{problemRow as T,replaceEntityProblemsForCodesBatchTx as E}from"../database/problems.js";import"../search/postings/postings-store.js";import{createSearchStore as D}from"../search/store.js";import{loadContentLayerState as ee}from"../context-state.js";import{replaceEntityRouteQueryDepsTx as O}from"../database/route-query-deps.js";import{SEARCH_PROBLEM_CODES as k,searchFacetInvalidProblem as A,searchableFailedProblem as j}from"../problems/producers.js";import{loadAttachedRouteSlugs as M}from"../database/route-deps.js";import{toRbacTeams as N,toRbacTeamsWithinRoute as P}from"./rbac-teams.js";import{buildDocumentFacets as F}from"../search/facet-rows.js";import{createHookEntityStore as I}from"./hook-inputs.js";import{and as L,eq as R,inArray as z}from"drizzle-orm";function te(t,n,r,i){let a=[...t].sort(),o=n.map(e=>[e.name,e.slug]);return(t,n)=>{let s=[2,t,a,o,r];return n&&i!==void 0&&s.push(i),e(s)}}async function B(e){let{context:t,database:n}=e,r=t.logger.child(`content-layer:search`),a=[],o=/* @__PURE__ */ new Map,s=new Map(t.pluginRegistry.listEntityLoaders().filter(e=>e.definition.searchable!==void 0||e.definition.markdown!==void 0).map(e=>[e.definition.id,e])),c=e.inlineSearchables??/* @__PURE__ */ new Map,l=e.entityKeys===void 0?void 0:[...e.entityKeys,...c.keys()],u=await re(n,l);if(s.size===0&&c.size===0&&u.size===0)return{synced:0,problems:a};let d=await ee({context:t,database:n}),f=D(n),m=g(d,t.config),h=y(t.config),_=q(d,m,h),v=_(),x=J(t),S=C(t.config),w=te(x,h,S.settings,b()),E=X(r,{configuredFields:x}),O=0,A=(e,t)=>{a.push(...t),o.set(e,t.map(t=>T(t,{scope:`entity`,entityId:e})))},N=(e,t,n,i)=>{let s=i instanceof Error?i.message:String(i);r.warn(`searchable failed for "${t}"; keeping its last documents: ${s}`);let c=j({entityKey:t,pluginId:n,message:s});a.push(c),o.set(e,[T(c,{scope:`entity`,entityId:e})])},P=async()=>(await ne(n,o,k,r),{synced:O,problems:a});l===void 0&&await U({context:t,database:n,state:d,store:f,stored:u,searchableLoaders:s});let F=[];for(let[e,n]of c){let i=d.entitiesByKey.get(e);if(i&&i.kind===`inline`){if(n===void 0){u.has(e)&&await f.removeSource(e);continue}try{let a=await E.prepare(e,n.docs,v);F.push({entityKey:e,docs:a.docs,...Y(e,n,a.docs),sourceHash:w(i.dataHash??``,!1),pluginVersion:t.pluginRegistry.getById(i.pluginId)?.version??``}),A(i.id,[...H(e,n.problems,r),...a.problems])}catch(t){N(i.id,e,i.pluginId,t)}}}if(await f.replaceSourceDocuments(F),O+=F.length,s.size===0)return P();let I=await ie(n,[...s.keys()],e.entityKeys),L=await ae(n,m,I.map(e=>e.id)),R=I.filter(e=>{let n=s.get(e.loaderId),r=n?t.pluginRegistry.getById(n.pluginId)?.version:void 0,i=u.get(e.key);return i===void 0||i.sourceHash!==w(e.dataHash??``,n?.definition.usesProjectMarkdocSchema===!0)||i.pluginVersion!==r||L.has(e.id)});if(R.length===0)return P();let z=await M(n,R.map(e=>e.key)),B=0,G=0,Z=0,Q=0,$=0;for(let r of p(R,50)){B+=1;let i=Date.now(),a=await K(n,r);Z+=Date.now()-i,i=Date.now();let o=[];for(let i of r){let r=s.get(i.loaderId);if(r){try{let e=await W({row:i,entry:r,value:a.get(i.id),attachedSlug:z.get(i.key),context:t,database:n,state:d,evaluator:m,resolveAccess:_,prepare:E.prepare,stampOf:w,allowsMarkdownCopy:S.allows});o.push(e),A(i.id,e.problems)}catch(e){N(i.id,i.key,r.pluginId,e)}G+=1,e.onProgress?.(G,R.length)}}Q+=Date.now()-i,i=Date.now(),await V({context:t,database:n,store:f,outputs:o}),$+=Date.now()-i,O+=o.length}return l===void 0&&r.info(`Synced ${R.length} content ${R.length===1?`item`:`items`} for search in ${B} ${B===1?`chunk`:`chunks`} (load ${i(Z)}, producers ${i(Q)}, writes ${i($)})`),P()}async function ne(e,t,n,r){if(t.size!==0)try{await e.db.transaction(e=>E(e,t,n))}catch(e){r.warn(`could not store the search problems of this pass`,{error:e instanceof Error?e.message:String(e)})}}async function V(e){let{context:t,outputs:n}=e;n.length!==0&&await e.database.db.transaction(async i=>{await e.store.replaceSourceDocuments(n.map(e=>({entityKey:e.row.key,docs:e.docs,fragments:e.fragments,uses:e.uses,sourceHash:e.sourceHash,pluginVersion:t.pluginRegistry.getById(e.entry.pluginId)?.version??``})),i),await O(i,{hook:`searchable`,scopes:n.map(e=>({originPluginId:e.entry.pluginId,entityId:e.row.id,dependencies:_(e.reads)}))});for(let e of n){if(await i.delete(c).where(R(c.entityKey,e.row.key)),e.markdownDocs.length===0)continue;let n=t.pluginRegistry.getById(e.entry.pluginId)?.version??``;for(let t of e.markdownDocs){let a=await Promise.all(t.chunks.map(async(i,a)=>{let o=t.headings?.[a]??null;return{entityKey:e.row.key,chunkIndex:a,routeSlug:t.routeSlug,text:i,heading:o?.heading??null,anchor:o?.anchor??null,title:t.title??null,description:t.description??null,rbacTeams:t.rbacTeams,hash:await r(new TextEncoder().encode(i)),sourceHash:e.sourceHash,pluginVersion:n}}));await i.insert(c).values(a)}}})}function H(e,t,n){let r=[],i=/* @__PURE__ */ new Set;for(let e of t??[])k.includes(e.code)?r.push(w(e)):i.add(e.code);for(let t of i)n.warn(`searchable for "${e}" reported problem code "${t}", which the search sync does not own; use one of ${k.join(`, `)}`);return r}async function U(e){for(let t of e.stored.keys()){let n=e.state.entitiesByKey.get(t);n&&(e.context.pluginRegistry.getById(n.pluginId)===void 0||n.kind!==`inline`&&(n.loaderId===null||!e.searchableLoaders.has(n.loaderId)))&&(await e.store.removeSource(t),await e.database.db.delete(c).where(R(c.entityKey,t)))}}async function W(e){let{row:t,entry:n,database:r}=e,i=n.definition.searchable,a=n.definition.markdown,o=e.value;if(!o||o.data===null||o.dataHash===null)throw Error(`Entity "${t.key}" has no stored value.`);let s=e.stampOf(o.dataHash,n.definition.usesProjectMarkdocSchema),c=await f(o.data,e=>h(r,e)),l=[],u=e=>l.push(e),d=e.evaluator.index(u),p=e.attachedSlug===void 0?void 0:d.get(e.attachedSlug),m=e.resolveAccess(u),g={config:e.context.config,entities:I(e.context,r,e.state,u),logger:e.context.logger.child(`content-layer:searchable:${n.definition.id}`),routes:d,route:p},_=i?await i(c,g):{docs:[]},v=a&&e.allowsMarkdownCopy(p)?await a(c,g):null,y=G(t.key,v,e.attachedSlug).map(e=>Object.assign(e,{rbacTeams:m.rbacTeamsForSlug(e.routeSlug)})),b=await e.prepare(t.key,_.docs??[],m);return{row:t,entry:n,docs:b.docs,...Y(t.key,_,b.docs),markdownDocs:y,sourceHash:s,reads:l,problems:[...H(t.key,_.problems,e.context.logger),...b.problems]}}function G(e,t,n){if(t===null)return[];if(!Array.isArray(t))throw Error(`markdown for "${e}" must return an array or null.`);let r=t.length>0&&typeof t[0]==`string`?[{chunks:t}]:t,i=/* @__PURE__ */ new Set,a=[];for(let t of r){let r=t.routeSlug??n;if(r!==void 0){if(!Array.isArray(t.chunks)||t.chunks.some(e=>typeof e!=`string`))throw Error(`markdown for "${e}" returned a document without string chunks.`);if(t.chunks.length!==0){if(i.has(r))throw Error(`markdown for "${e}" returned "${r}" twice.`);i.add(r),a.push({routeSlug:r,chunks:t.chunks,headings:t.headings,...typeof t.title==`string`&&t.title!==``?{title:t.title}:{},...typeof t.description==`string`&&t.description!==``?{description:t.description}:{}})}}}return a}async function K(e,t){let n=await e.db.select({id:s.id,data:s.data,dataHash:s.dataHash}).from(s).where(z(s.id,t.map(e=>e.id)));return new Map(n.map(e=>[e.id,e]))}function q(e,t,n){let r=[...e.routeBySlug.values()].filter(e=>e.wildcardPrefix),i=t=>{if(e.routeBySlug.has(t))return t;let n=new Set(d(t)),i;for(let e of r)n.has(e.slug)&&(i===void 0||e.slug.length>i.length)&&(i=e.slug);return i};return r=>{let a=t.index(r),o=e=>a.getAccess(i(e)??e),s=t=>e.routeBySlug.get(i(t)??t);return{accessForSlug:o,rbacTeamsForSlug:e=>N(o(e)),localeForSlug:e=>s(e)?.locale??``,originFilePathForSlug:e=>s(e)?.originFilePath??null,versionForSlug:e=>{let t=s(e);return t?.versionGroup!=null&&t.version!=null?{group:t.versionGroup,version:t.version}:null},productForSlug:e=>{let t=s(e),r=t?t.canonicalSlug||t.slug:e;return S(n,r)}}}}function J(e){return new Set(n(e.config).map(e=>e.field).filter(e=>e!==t))}function Y(e,t,n){let r=t.fragments??[],i=t.uses??[];if(!Array.isArray(r)||!Array.isArray(i))throw Error(`searchable for "${e}" must return fragments and uses as arrays.`);for(let t of r)if(typeof t.fragmentKey!=`string`||typeof t.name!=`string`)throw Error(`searchable for "${e}" returned a fragment without a key or name.`);let a=new Set(n.map(e=>e.docKey));return{fragments:r,uses:i.filter(e=>a.has(e.docKey))}}function X(e,t){return{async prepare(n,r,i){if(!Array.isArray(r))throw Error(`searchable for "${n}" must return { docs: SearchDocument[] }.`);let a=/* @__PURE__ */ new Set,s=/* @__PURE__ */ new Set,c=[],l=[];for(let u of r){if(Q(n,u),a.has(u.docKey))throw Error(`searchable for "${n}" returned docKey "${u.docKey}" twice.`);a.add(u.docKey);let r=o(u.routeSlug),d=i.accessForSlug(r);if(d===void 0){e.warn(`search document "${u.docKey}" of "${n}" dropped: no route "${r}"`);continue}let{facets:f,rbacScope:p,keywords:m,...h}=u,g=i.productForSlug(r),_=i.versionForSlug(r),v=F({facets:f,configuredFields:t.configuredFields,groupKey:u.groupKey,teams:P(d,p),version:_,product:g?.name??null,keywords:x(m),onDropped:(e,t)=>{let a=`${e}\u0000${t}`;s.has(a)||(s.add(a),c.push(A({entityKey:n,field:e,reason:Z[t],filePath:i.originFilePathForSlug(r)})))}}),y={...h,routeSlug:r,locale:i.localeForSlug(r),facets:v};g&&(y.product={name:g.name,folder:g.folder}),_&&(y.version=_),l.push(y)}return{docs:l,problems:c}}}}const Z={"not-an-object":`facets must be an object of field to string or string list`,reserved:`fields starting with "redocly_" are derived by the engine`,invalid:`a value must be a string, number, boolean, or a list of those`,"too-long":`a value must be at most 500 characters`,"too-many-words":`a keyword must have at most 10 words`};function Q(e,t){let n=t;if(!(typeof n==`object`&&n&&typeof n.docKey==`string`&&n.docKey!==``&&typeof n.routeSlug==`string`&&typeof n.title==`string`&&typeof n.text==`string`))throw Error(`searchable for "${e}" returned an invalid document (docKey, routeSlug, title, text are required strings).`);if(n.rbacScope!==void 0&&!a(n.rbacScope))throw Error(`searchable for "${e}" returned document "${n.docKey}" with an invalid rbacScope.`)}async function re(e,t){let n=/* @__PURE__ */ new Map;for(let r of t===void 0?[void 0]:p(t,m(void 0))){let t=await e.db.selectDistinct({entityKey:u.entityKey,sourceHash:u.sourceHash,pluginVersion:u.pluginVersion}).from(u).where(r===void 0?void 0:z(u.entityKey,[...r]));for(let{entityKey:e,...r}of t)n.set(e,r);let i=await e.db.selectDistinct({entityKey:c.entityKey,sourceHash:c.sourceHash,pluginVersion:c.pluginVersion}).from(c).where(r===void 0?void 0:z(c.entityKey,[...r]));for(let{entityKey:e,...t}of i)n.has(e)||n.set(e,t)}return n}async function ie(e,t,n){return n!==void 0&&n.length===0?[]:(await e.db.select({id:s.id,key:s.key,loaderId:s.loaderId,dataHash:s.dataHash}).from(s).where(L(R(s.status,`ready`),R(s.isCurrent,1),z(s.loaderId,[...t]),n===void 0?void 0:z(s.key,[...n]))).orderBy(s.key)).filter(e=>e.loaderId!==null)}async function ae(e,t,n){let r=/* @__PURE__ */ new Set;if(n.length===0)return r;let i=new Set(n),a=n.length<=m(void 0),o=await e.db.select({entityId:l.entityId,kind:l.kind,argument:l.argument,resultHash:l.resultHash}).from(l).where(L(R(l.hook,`searchable`),a?z(l.entityId,[...n]):void 0));for(let e of o)e.entityId!==null&&i.has(e.entityId)&&!r.has(e.entityId)&&v(t,e)&&r.add(e.entityId);return r}function $(e,t){for(let n of e)n.kind===`inline`&&t.set(n.key,n.searchable)}export{$ as collectInlineSearchables,B as runSearchSync};
|
|
1
|
+
import{hashJsonValue as e}from"../database/utils/json.js";import{SEARCH_GROUP_FACET_FIELD as t,resolveSearchFacets as n}from"../../shared/search/facets.js";import{hashContentBytes as r}from"../../shared/utils/content-hash.js";import{formatDuration as i}from"../../shared/log-format.js";import{isRbacScope as a}from"../../shared/rbac/engine.js";import{normalizeRouteSlug as o}from"../../shared/utils/routes/normalize-route-slug.js";import{entities as s,pageMarkdown as c,routeQueryDeps as l,searchDocuments as u}from"../database/schema.js";import{computePathPrefixes as d}from"../../shared/utils/routes/compute-path-prefixes.js";import{replaceDataRefMarkers as f}from"../materialization/data-ref.js";import{batchValues as p,normalizeBatchSize as m}from"../database/connection.js";import{resolveDataRef as h}from"../database/blobs.js";import{createRouteQueryEvaluator as g,dedupeRouteQueryDependencies as _,isRouteQueryStale as v}from"../plugins/utils/route-index.js";import{listConfiguredProducts as y}from"../products.js";import{userMarkdocSchemaDigest as b}from"../markdown/constants/tag-sets.js";import{curationKeywordsOf as x}from"../search/keywords.js";import{findProductForSlug as S}from"../../shared/products.js";import{createMarkdownCopyRule as C}from"../markdown-copy-rule.js";import{createProblem as w}from"../problems/codes.js";import{problemRow as T,replaceEntityProblemsForCodesBatchTx as E}from"../database/problems.js";import"../search/postings/postings-store.js";import{createSearchStore as D}from"../search/store.js";import{replaceEntityRouteQueryDepsTx as O}from"../database/route-query-deps.js";import{SEARCH_PROBLEM_CODES as k,searchFacetInvalidProblem as A,searchableFailedProblem as j}from"../problems/producers.js";import{loadAttachedRouteSlugs as M}from"../database/route-deps.js";import{toRbacTeams as N,toRbacTeamsWithinRoute as P}from"./rbac-teams.js";import{buildDocumentFacets as F}from"../search/facet-rows.js";import{createHookEntityStore as I}from"./hook-inputs.js";import{and as L,eq as R,inArray as z}from"drizzle-orm";function B(t,n,r,i){let a=[...t].sort(),o=n.map(e=>[e.name,e.slug]);return(t,n)=>{let s=[2,t,a,o,r];return n&&i!==void 0&&s.push(i),e(s)}}async function V(e){let{context:t,database:n}=e,r=t.logger.child(`content-layer:search`),a=[],o=/* @__PURE__ */ new Map,s=new Map(t.pluginRegistry.listEntityLoaders().filter(e=>e.definition.searchable!==void 0||e.definition.markdown!==void 0).map(e=>[e.definition.id,e])),c=e.inlineSearchables??/* @__PURE__ */ new Map,l=e.entityKeys===void 0?void 0:[...e.entityKeys,...c.keys()],u=await re(n,l);if(s.size===0&&c.size===0&&u.size===0)return{synced:0,problems:a};let{materializer:d}=e,{state:f}=d,m=D(n),h=g(f,t.config),_=y(t.config),v=ee(f,h,_),x=v(),S=te(t),w=C(t.config),E=B(S,_,w.settings,b()),O=ne(r,{configuredFields:S}),A=0,N=(e,t)=>{a.push(...t),o.set(e,t.map(t=>T(t,{scope:`entity`,entityId:e})))},P=(e,t,n,i)=>{let s=i instanceof Error?i.message:String(i);r.warn(`searchable failed for "${t}"; keeping its last documents: ${s}`);let c=j({entityKey:t,pluginId:n,message:s});a.push(c),o.set(e,[T(c,{scope:`entity`,entityId:e})])},F=async()=>(await H(n,o,k,r),{synced:A,problems:a});l===void 0&&await G({context:t,database:n,state:f,store:m,stored:u,searchableLoaders:s});let I=[];for(let[e,n]of c){let i=f.entitiesByKey.get(e);if(i&&i.kind===`inline`){if(n===void 0){u.has(e)&&await m.removeSource(e);continue}try{let a=await O.prepare(e,n.docs,x);I.push({entityKey:e,docs:a.docs,...Y(e,n,a.docs),sourceHash:E(i.dataHash??``,!1),pluginVersion:t.pluginRegistry.getById(i.pluginId)?.version??``}),N(i.id,[...W(e,n.problems,r),...a.problems])}catch(t){P(i.id,e,i.pluginId,t)}}}if(await m.replaceSourceDocuments(I),A+=I.length,s.size===0)return F();let L=await ie(n,[...s.keys()],e.entityKeys),R=await ae(n,h,L.map(e=>e.id)),z=L.filter(e=>{let n=s.get(e.loaderId),r=n?t.pluginRegistry.getById(n.pluginId)?.version:void 0,i=u.get(e.key);return i===void 0||i.sourceHash!==E(e.dataHash??``,n?.definition.usesProjectMarkdocSchema===!0)||i.pluginVersion!==r||R.has(e.id)});if(z.length===0)return F();let V=await M(n,z.map(e=>e.key)),q=0,X=0,Z=0,Q=0,$=0;for(let r of p(z,50)){q+=1;let i=Date.now(),a=await J(n,r);Z+=Date.now()-i,i=Date.now();let o=[];for(let i of r){let r=s.get(i.loaderId);if(r){try{let e=await K({row:i,entry:r,value:a.get(i.id),attachedSlug:V.get(i.key),context:t,database:n,materializer:d,evaluator:h,resolveAccess:v,prepare:O.prepare,stampOf:E,allowsMarkdownCopy:w.allows});o.push(e),N(i.id,e.problems)}catch(e){P(i.id,i.key,r.pluginId,e)}X+=1,e.onProgress?.(X,z.length)}}Q+=Date.now()-i,i=Date.now(),await U({context:t,database:n,store:m,outputs:o}),$+=Date.now()-i,A+=o.length}return l===void 0&&r.info(`Synced ${z.length} content ${z.length===1?`item`:`items`} for search in ${q} ${q===1?`chunk`:`chunks`} (load ${i(Z)}, producers ${i(Q)}, writes ${i($)})`),F()}async function H(e,t,n,r){if(t.size!==0)try{await e.db.transaction(e=>E(e,t,n))}catch(e){r.warn(`could not store the search problems of this pass`,{error:e instanceof Error?e.message:String(e)})}}async function U(e){let{context:t,outputs:n}=e;n.length!==0&&await e.database.db.transaction(async i=>{await e.store.replaceSourceDocuments(n.map(e=>({entityKey:e.row.key,docs:e.docs,fragments:e.fragments,uses:e.uses,sourceHash:e.sourceHash,pluginVersion:t.pluginRegistry.getById(e.entry.pluginId)?.version??``})),i),await O(i,{hook:`searchable`,scopes:n.map(e=>({originPluginId:e.entry.pluginId,entityId:e.row.id,dependencies:_(e.reads)}))});for(let e of n){if(await i.delete(c).where(R(c.entityKey,e.row.key)),e.markdownDocs.length===0)continue;let n=t.pluginRegistry.getById(e.entry.pluginId)?.version??``;for(let t of e.markdownDocs){let a=await Promise.all(t.chunks.map(async(i,a)=>{let o=t.headings?.[a]??null;return{entityKey:e.row.key,chunkIndex:a,routeSlug:t.routeSlug,text:i,heading:o?.heading??null,anchor:o?.anchor??null,title:t.title??null,description:t.description??null,rbacTeams:t.rbacTeams,hash:await r(new TextEncoder().encode(i)),sourceHash:e.sourceHash,pluginVersion:n}}));await i.insert(c).values(a)}}})}function W(e,t,n){let r=[],i=/* @__PURE__ */ new Set;for(let e of t??[])k.includes(e.code)?r.push(w(e)):i.add(e.code);for(let t of i)n.warn(`searchable for "${e}" reported problem code "${t}", which the search sync does not own; use one of ${k.join(`, `)}`);return r}async function G(e){for(let t of e.stored.keys()){let n=e.state.entitiesByKey.get(t);n&&(e.context.pluginRegistry.getById(n.pluginId)===void 0||n.kind!==`inline`&&(n.loaderId===null||!e.searchableLoaders.has(n.loaderId)))&&(await e.store.removeSource(t),await e.database.db.delete(c).where(R(c.entityKey,t)))}}async function K(e){let{row:t,entry:n,database:r}=e,i=n.definition.searchable,a=n.definition.markdown,o=e.value;if(!o||o.data===null||o.dataHash===null)throw Error(`Entity "${t.key}" has no stored value.`);let s=e.stampOf(o.dataHash,n.definition.usesProjectMarkdocSchema),c=await f(o.data,e=>h(r,e)),l=[],u=e=>l.push(e),d=e.evaluator.index(u),p=e.attachedSlug===void 0?void 0:d.get(e.attachedSlug),m=e.resolveAccess(u),g={config:e.context.config,entities:I(r,e.materializer,u),logger:e.context.logger.child(`content-layer:searchable:${n.definition.id}`),routes:d,route:p},_=i?await i(c,g):{docs:[]},v=a&&e.allowsMarkdownCopy(p)?await a(c,g):null,y=q(t.key,v,e.attachedSlug).map(e=>Object.assign(e,{rbacTeams:m.rbacTeamsForSlug(e.routeSlug)})),b=await e.prepare(t.key,_.docs??[],m);return{row:t,entry:n,docs:b.docs,...Y(t.key,_,b.docs),markdownDocs:y,sourceHash:s,reads:l,problems:[...W(t.key,_.problems,e.context.logger),...b.problems]}}function q(e,t,n){if(t===null)return[];if(!Array.isArray(t))throw Error(`markdown for "${e}" must return an array or null.`);let r=t.length>0&&typeof t[0]==`string`?[{chunks:t}]:t,i=/* @__PURE__ */ new Set,a=[];for(let t of r){let r=t.routeSlug??n;if(r!==void 0){if(!Array.isArray(t.chunks)||t.chunks.some(e=>typeof e!=`string`))throw Error(`markdown for "${e}" returned a document without string chunks.`);if(t.chunks.length!==0){if(i.has(r))throw Error(`markdown for "${e}" returned "${r}" twice.`);i.add(r),a.push({routeSlug:r,chunks:t.chunks,headings:t.headings,...typeof t.title==`string`&&t.title!==``?{title:t.title}:{},...typeof t.description==`string`&&t.description!==``?{description:t.description}:{}})}}}return a}async function J(e,t){let n=await e.db.select({id:s.id,data:s.data,dataHash:s.dataHash}).from(s).where(z(s.id,t.map(e=>e.id)));return new Map(n.map(e=>[e.id,e]))}function ee(e,t,n){let r=[...e.routeBySlug.values()].filter(e=>e.wildcardPrefix),i=t=>{if(e.routeBySlug.has(t))return t;let n=new Set(d(t)),i;for(let e of r)n.has(e.slug)&&(i===void 0||e.slug.length>i.length)&&(i=e.slug);return i};return r=>{let a=t.index(r),o=e=>a.getAccess(i(e)??e),s=t=>e.routeBySlug.get(i(t)??t);return{accessForSlug:o,rbacTeamsForSlug:e=>N(o(e)),localeForSlug:e=>s(e)?.locale??``,originFilePathForSlug:e=>s(e)?.originFilePath??null,versionForSlug:e=>{let t=s(e);return t?.versionGroup!=null&&t.version!=null?{group:t.versionGroup,version:t.version}:null},productForSlug:e=>{let t=s(e),r=t?t.canonicalSlug||t.slug:e;return S(n,r)}}}}function te(e){return new Set(n(e.config).map(e=>e.field).filter(e=>e!==t))}function Y(e,t,n){let r=t.fragments??[],i=t.uses??[];if(!Array.isArray(r)||!Array.isArray(i))throw Error(`searchable for "${e}" must return fragments and uses as arrays.`);for(let t of r)if(typeof t.fragmentKey!=`string`||typeof t.name!=`string`)throw Error(`searchable for "${e}" returned a fragment without a key or name.`);let a=new Set(n.map(e=>e.docKey));return{fragments:r,uses:i.filter(e=>a.has(e.docKey))}}function ne(e,t){return{async prepare(n,r,i){if(!Array.isArray(r))throw Error(`searchable for "${n}" must return { docs: SearchDocument[] }.`);let a=/* @__PURE__ */ new Set,s=/* @__PURE__ */ new Set,c=[],l=[];for(let u of r){if(Z(n,u),a.has(u.docKey))throw Error(`searchable for "${n}" returned docKey "${u.docKey}" twice.`);a.add(u.docKey);let r=o(u.routeSlug),d=i.accessForSlug(r);if(d===void 0){e.warn(`search document "${u.docKey}" of "${n}" dropped: no route "${r}"`);continue}let{facets:f,rbacScope:p,keywords:m,...h}=u,g=i.productForSlug(r),_=i.versionForSlug(r),v=F({facets:f,configuredFields:t.configuredFields,groupKey:u.groupKey,teams:P(d,p),version:_,product:g?.name??null,keywords:x(m),onDropped:(e,t)=>{let a=`${e}\u0000${t}`;s.has(a)||(s.add(a),c.push(A({entityKey:n,field:e,reason:X[t],filePath:i.originFilePathForSlug(r)})))}}),y={...h,routeSlug:r,locale:i.localeForSlug(r),facets:v};g&&(y.product={name:g.name,folder:g.folder}),_&&(y.version=_),l.push(y)}return{docs:l,problems:c}}}}const X={"not-an-object":`facets must be an object of field to string or string list`,reserved:`fields starting with "redocly_" are derived by the engine`,invalid:`a value must be a string, number, boolean, or a list of those`,"too-long":`a value must be at most 500 characters`,"too-many-words":`a keyword must have at most 10 words`};function Z(e,t){let n=t;if(!(typeof n==`object`&&n&&typeof n.docKey==`string`&&n.docKey!==``&&typeof n.routeSlug==`string`&&typeof n.title==`string`&&typeof n.text==`string`))throw Error(`searchable for "${e}" returned an invalid document (docKey, routeSlug, title, text are required strings).`);if(n.rbacScope!==void 0&&!a(n.rbacScope))throw Error(`searchable for "${e}" returned document "${n.docKey}" with an invalid rbacScope.`)}async function re(e,t){let n=/* @__PURE__ */ new Map;for(let r of t===void 0?[void 0]:p(t,m(void 0))){let t=await e.db.selectDistinct({entityKey:u.entityKey,sourceHash:u.sourceHash,pluginVersion:u.pluginVersion}).from(u).where(r===void 0?void 0:z(u.entityKey,[...r]));for(let{entityKey:e,...r}of t)n.set(e,r);let i=await e.db.selectDistinct({entityKey:c.entityKey,sourceHash:c.sourceHash,pluginVersion:c.pluginVersion}).from(c).where(r===void 0?void 0:z(c.entityKey,[...r]));for(let{entityKey:e,...t}of i)n.has(e)||n.set(e,t)}return n}async function ie(e,t,n){return n!==void 0&&n.length===0?[]:(await e.db.select({id:s.id,key:s.key,loaderId:s.loaderId,dataHash:s.dataHash}).from(s).where(L(R(s.status,`ready`),R(s.isCurrent,1),z(s.loaderId,[...t]),n===void 0?void 0:z(s.key,[...n]))).orderBy(s.key)).filter(e=>e.loaderId!==null)}async function ae(e,t,n){let r=/* @__PURE__ */ new Set;if(n.length===0)return r;let i=new Set(n),a=n.length<=m(void 0),o=await e.db.select({entityId:l.entityId,kind:l.kind,argument:l.argument,resultHash:l.resultHash}).from(l).where(L(R(l.hook,`searchable`),a?z(l.entityId,[...n]):void 0));for(let e of o)e.entityId!==null&&i.has(e.entityId)&&!r.has(e.entityId)&&v(t,e)&&r.add(e.entityId);return r}function Q(e,t){for(let n of e)n.kind===`inline`&&t.set(n.key,n.searchable)}export{Q as collectInlineSearchables,V as runSearchSync};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{normalizeRouteSlug as e}from"../../../shared/utils/routes/normalize-route-slug.js";import{SYSTEM_ROUTES as t}from"../../../
|
|
1
|
+
import{normalizeRouteSlug as e}from"../../../shared/utils/routes/normalize-route-slug.js";import{SYSTEM_ROUTES as t}from"../../../shared/constants/system-routes.js";import{toSlugCollisionProblems as n,toWildcardShadowProblems as r}from"../../problems/producers.js";import{resolveSlugsAndHandleCollisions as i}from"./resolve-slugs-and-handle-collisions.js";const a=new Set(t.map(e));function o(e){return/* @__PURE__ */ new Set([...a,...e?.keys()??[]])}function s(e){let t=[...e.state.routesByOriginKey.values()].flat(),o=new Map([...a].map(e=>[e,null]));for(let[t,n]of e.reservedSlugs??[])o.has(t)||o.set(t,n);let{collisions:s,resolved:u,warnings:d}=i(t,o);for(let t of s)e.logger.debug(`Route slug collision — dev serves the first processed member`,{slug:t.slug,kind:t.kind,members:t.members.map(e=>`${e.originFilePath} (${e.originPluginId})`)});let f=new Set(d.map(c));l({warnings:d,previousShadowKeys:e.previousWildcardShadows,logger:e.logger});let p=new Map(s.map(e=>[e.slug,e]));return{collisions:p,routeBySlug:new Map(u.map(e=>[e.slug,e.candidate])),wildcardShadows:f,problems:[...n(p),...r(d)]}}function c(e){return`${e.slug} ${e.paths[1]??``}`}function l(e){for(let t of e.warnings)e.previousShadowKeys.has(c(t))||e.logger.debug(`Route slug conflict`,t)}export{a as SYSTEM_SLUGS,o as allReservedSlugs,s as resolveFinalSlugs};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{L10N_DIR as e}from"../../shared/constants/l10n/defaults.js";import{resolveContentPluginsForFile as t}from"../plugins/registry.js";import{hashContentConfigDependency as n}from"../config-deps.js";import{extractFileContributions as r}from"../plugins/execute.js";import{createSourceFile as i}from"../plugins/source-file.js";import{createContentOriginKey as a}from"../plugins/utils/origin-key.js";import{L10N_PLUGIN_ID as o}from"../plugins/internal/l10n/translations-loader.js";import"../plugins/internal/l10n/index.js";import"../plugins/index.js";import{createProblem as s}from"../problems/codes.js";import{garbageCollectOrphanedEntities as c,propagateRouteQueryInvalidation as l}from"../database/invalidation.js";import{problemRow as u}from"../database/problems.js";import{loadFileManifestsByPaths as d}from"../database/files.js";import{loadContentLayerState as f,reloadContentLayerEntities as p}from"../context-state.js";import{
|
|
1
|
+
import{L10N_DIR as e}from"../../shared/constants/l10n/defaults.js";import{resolveContentPluginsForFile as t}from"../plugins/registry.js";import{hashContentConfigDependency as n}from"../config-deps.js";import{extractFileContributions as r}from"../plugins/execute.js";import{createSourceFile as i}from"../plugins/source-file.js";import{createContentOriginKey as a}from"../plugins/utils/origin-key.js";import{L10N_PLUGIN_ID as o}from"../plugins/internal/l10n/translations-loader.js";import"../plugins/internal/l10n/index.js";import"../plugins/index.js";import{createProblem as s}from"../problems/codes.js";import{garbageCollectOrphanedEntities as c,propagateRouteQueryInvalidation as l}from"../database/invalidation.js";import{problemRow as u}from"../database/problems.js";import{loadFileManifestsByPaths as d}from"../database/files.js";import{loadContentLayerState as f,reloadContentLayerEntities as p}from"../context-state.js";import{resolveRevisionsConfig as m}from"../revisions/config.js";import{bulkSyncOrigins as ee}from"../database/origins.js";import{extractFailedProblem as h,sluggerProblemSources as g}from"../problems/producers.js";import{createEntityMaterializer as _}from"../materialization/materializer.js";import{finalizeHookInputs as te,hasAnyPluginHookTrigger as ne}from"./hook-inputs.js";import{collectInlineSearchables as v}from"./search-sync.js";import{dropCollidingRoutes as y}from"./slugger/drop-colliding-routes.js";import{allReservedSlugs as b,resolveFinalSlugs as x}from"./slugger/resolve-final-slugs.js";import{normalizePluginArtifacts as S}from"./normalize-artifacts.js";import{runFinalizePhase as C}from"./finalize-phase.js";import{applyOriginRoutesToWorkingState as w,cloneContentLayerState as T,removeFileFromWorkingState as E,removeOriginFromWorkingState as D}from"./working-state.js";import{dropCollidingExtractAllRoutes as O,runExtractAllPhase as k}from"./extract-all-phase.js";import{diffFallbackRoutes as re,generateFallbackRoutes as ie}from"./generate-fallback-routes.js";async function A(n){let s=await f({context:n.context,database:n.database}),l=T(s);for(let e of n.removedPaths)E(l,e);if(n.files.length===0&&n.removedPaths.size===0&&![`extractAll`,`finalize`].some(e=>ne(e,n.context,s,n.pluginScopedOriginRows,n.routeQueryRows,n.hookState,e===`finalize`?te(n.localeContext):void 0)))return await N(n,l),{state:l,inlineSearchables:/* @__PURE__ */ new Map,routeChanges:{changedSlugs:[],unbounded:!1}};let h=n.context.logger.child(`content-layer:sync`),{localeContext:S}=n,A=F(n.context.config,`/l10n`),j=F(n.context.config,`/metadataGlobs`),M=m(n.context.config),P=n.revisions,L=()=>_({deps:{...n.context,state:l},database:n.database,revisions:P}),R=await k({context:n.context,database:n.database,materializer:L(),files:n.files,originRows:n.pluginScopedOriginRows,routeQueryRows:n.routeQueryRows,hookState:n.hookState,logger:n.context.logger.child(`content-layer:extractAll`)}),z=n.context.logger.child(`content-layer:plugin`),B=n.files.filter(e=>e.manifest===void 0).map(e=>e.path),V=await d(n.database,B),H=n.context.pluginRegistry.list().map(e=>e.id);D(l,H,o,null);let U=[],W=`${e}/`,G=n.files.map(e=>({file:e,localized:Number(e.path.startsWith(W))})).sort((e,t)=>e.localized-t.localized).map(e=>e.file);for(let[e,a]of G.entries()){let o=l.filesByPath.get(a.path)??l.ignoredFilesByPath.get(a.path);if(!o)throw Error(`Missing stored file state for "${a.path}".`);let s=t(n.context,a.path),c=(l.filePluginsByPath.get(a.path)??[]).map(e=>e.pluginId).filter(e=>!s.some(t=>t.id===e)),u=[];for(let e of c)D(l,H,e,a.path),u.push({hook:`extract`,originDependencies:[],originFileId:o.id,originFilePath:a.path,originPluginId:e,routeDependencies:[],routes:[],entities:[],problems:[]});for(let e of s)D(l,H,e.id,a.path);let d=i({path:a.path,hash:a.hash,manifest:a.manifest??V.get(a.path)??{}}),f=await r(s,d,{config:n.context.config,fs:n.context.platform.fs,contentDir:n.context.contentDir,scannedPaths:()=>l.filesByPath.keys(),logger:z});for(let e of f)I({context:n.context,workingState:l,allPluginIds:H,filePath:a.path,fileRecord:o,attempt:e,origins:u,revisions:M,localeContext:S,l10nConfigDependency:A,metadataGlobsConfigDependency:j,pluginLogger:z});let p=s.map(e=>({id:e.id,version:e.version}));U.push({fileId:o.id,filePath:a.path,plugins:p,origins:u}),l.filePluginsByPath.set(a.path,p.map(e=>({fileId:o.id,pluginId:e.id,pluginVersion:e.version}))),n.onFileSynced?.(e+1,G.length)}let K=n.context.pluginRegistry.listReservedSlugs(),{collisions:q,routeBySlug:oe,wildcardShadows:se,problems:J}=x({state:l,previousWildcardShadows:s.wildcardShadows,reservedSlugs:K,logger:h});l.wildcardShadows=se,l.routeBySlug=oe;let ce=/* @__PURE__ */ new Set([...n.files.map(e=>e.path),...n.removedPaths,...R.pending.map(e=>e.pluginId),...R.failures.map(e=>e.pluginId)]),Y=new Set(J.map(e=>e.key));for(let e of n.storedSluggerProblems){if(e.key===null)continue;let t=g(e.data);t.length>0&&t.every(e=>ce.has(e))&&Y.add(e.key)}y({state:l,bulkItems:U,collisions:q,allPluginIds:H}),O(R.pending,q);let X=ie({state:l,localeContext:S,reservedSlugs:b(K)}),le=re(s.routesByOriginKey.get(a(`internal.l10n`,null))??[],X);w(l,H,o,null,X);let Z=/* @__PURE__ */ new Map;for(let e of U)for(let t of e.origins)v(t.entities,Z);for(let e of R.pending)v(e.entities,Z);let Q=/* @__PURE__ */ new Set;if(P){for(let e of U)if(P.isTracked(e.filePath))for(let t of e.origins)for(let e of t.entities)Q.add(e.key)}await ee(n.database,U,{logger:z,revisions:P?{max:P.max,trackedKeys:Q}:void 0,extractAllOrigins:R.pending.map(e=>({hook:`extractAll`,originFileId:null,originFilePath:null,originPluginId:e.pluginId,originDependencies:e.originDependencies,routeDependencies:e.routeDependencies,routes:e.routes,entities:e.entities,problems:[],fingerprint:e.inputsHash,routeQueries:e.routeQueries})),extractAllFailures:R.failures,globalProblems:{source:`slugger`,keys:[...Y],newProblems:J.map(e=>u(e.problem,{scope:`global`,source:`slugger`,key:e.key}))},fallbackRoutes:{pluginId:o,...le}}),await p({state:l,database:n.database}),await N(n,l);let $=await C({context:n.context,database:n.database,localeContext:S,materializer:L(),files:n.files,pluginScopedOriginRows:n.pluginScopedOriginRows,routeQueryRows:n.routeQueryRows,hookState:n.hookState,logger:n.context.logger.child(`content-layer:finalize`),collectEntities:e=>v(e,Z)}),ue=await c(n.database);return($.ranPluginIds.length>0||ue.removedOrphans)&&await p({state:l,database:n.database}),{state:l,inlineSearchables:Z,routeChanges:ae(s.routeBySlug,l.routeBySlug)}}function ae(e,t){let n=[];for(let[r,i]of e){let e=t.get(r);(e===void 0||e.originFilePath!==i.originFilePath)&&n.push(r)}for(let r of t.keys())e.has(r)||n.push(r);return{changedSlugs:n,unbounded:!j(e,t)}}function j(e,t){let n=M(e),r=M(t);return n.size===r.size&&[...n].every(e=>r.has(e))}function M(e){let t=/* @__PURE__ */ new Set;for(let[n,r]of e)r.wildcardPrefix&&t.add(n);return t}async function N(e,t){let{invalidatedKeys:n}=await l(e.database,{rows:e.routeQueryRows,state:t,config:e.context.config,logger:e.context.logger.child(`content-layer:db`)});for(let e of n){let n=t.entitiesByKey.get(e);n&&(n.status=`stale`)}}function P(e){return[...e.configDependencies.map(e=>({...e,source:`config`})),...e.fileDependencies]}function F(e,t){return{source:`config`,accessType:`value`,path:t,valueHash:n(e,{accessType:`value`,path:t})}}function I(e){let{context:t,workingState:n,allPluginIds:r,filePath:i,fileRecord:o,attempt:c}=e,l=c.result.ok?c.result.value:null,u=e.localeContext.isL10nPath(i)?[...P(c),e.l10nConfigDependency]:P(c);if(!l){e.origins.push({hook:`extract`,originDependencies:u,originFileId:o.id,originFilePath:i,originPluginId:c.plugin.id,routeDependencies:[],routes:[],entities:[],problems:c.result.ok?[]:[h({filePath:i,pluginId:c.plugin.id,message:c.result.error.message})]});return}let d=S({context:t,filePath:i,pluginId:c.plugin.id,revisions:e.revisions,routeDrafts:l.routes,standaloneEntities:l.entities??[],localeContext:e.localeContext,resolveCanonicalRoute:(e,t)=>n.routesByOriginKey.get(a(c.plugin.id,e))?.find(e=>e.routeId===t)});w(n,r,c.plugin.id,i,d.routes),e.origins.push({hook:`extract`,originDependencies:d.routes.length>0?[...u,e.metadataGlobsConfigDependency]:u,originFileId:o.id,originFilePath:i,originPluginId:c.plugin.id,routeDependencies:d.routeDependencies,routes:d.routes,entities:d.entities,problems:(l.problems??[]).map(e=>s({...e,location:e.location??{filePath:i}})),linkRefs:l.linkRefs})}export{A as syncContentFiles};
|