hammerkit 1.27.0-next.8 → 2.0.0-next.1

package/dist/kubernetes/kubernetes-instance.js

@@ -24,6 +24,7 @@ function createKubernetesInstances(kubernetes) {
         return {
             objectApi,
             log: new client_node_1.Log(kc),
+            metrics: new client_node_1.Metrics(kc),
             exec,
             appsApi,
             coreApi,

package/dist/kubernetes/network-policy.js

@@ -17,7 +17,7 @@ const sleep_1 = require("../utils/sleep");
 const network_gate_1 = require("./network-gate");
 const job_placement_1 = require("./job-placement");
 Object.defineProperty(exports, "RUN_LABEL", { enumerable: true, get: function () { return job_placement_1.RUN_LABEL; } });
-const ID_LABEL = 'hammerkit.dev/id';
+const resources_1 = require("./resources");
 const PROXY_LABEL = 'hammerkit.dev/proxy-for';
 const NETWORK_OF_LABEL = 'hammerkit.dev/network-of';
 // on the proxy of a task on a sandbox: the sandbox, whose pod of files it stays with
@@ -86,7 +86,7 @@ function getNetworkPolicyName(item, scope) {
 exports.getNetworkPolicyName = getNetworkPolicyName;
 // the pods of the item, of the run when the scope has one
 function getItemPodLabels(item, scope) {
-    return { [ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
+    return { [resources_1.ID_LABEL]: item.id(), ...(scope.run ? { [job_placement_1.RUN_LABEL]: scope.run } : {}) };
 }
 function toSelector(labels) {
     return Object.entries(labels)
@@ -107,7 +107,12 @@ function getIngress(env, item) {
     if (ids.length === 0) {
         return [];
     }
-    const from = { matchExpressions: [{ key: ID_LABEL, operator: 'In', values: ids }] };
+    // on a server, of this run: the runs of a caller share their namespace
+    const run = (0, job_placement_1.getRunLabels)(env);
+    const from = {
+        matchExpressions: [{ key: resources_1.ID_LABEL, operator: 'In', values: ids }],
+        ...(Object.keys(run).length > 0 ? { matchLabels: run } : {}),
+    };
     // the client's name for `from`, a reserved word
     return [{ _from: [{ podSelector: from }] }];
 }
@@ -124,7 +129,8 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
             namespace: env.namespace,
             name: getNetworkPolicyName(item, scope),
             annotations: { 'hammerkit.dev/version': (0, version_1.getVersion)() },
-            labels: { [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) },
+            labels: { [NETWORK_OF_LABEL]: item.id(), ...(scope.owners ? {} : (0, job_placement_1.getRunLabels)(env)) },
+            ...(scope.owners ? { ownerReferences: scope.owners } : {}),
         },
         spec: {
             podSelector: { matchLabels: getItemPodLabels(item, scope) },
@@ -132,7 +138,7 @@ function buildNetworkPolicy(env, item, network, scope, proxy = getRunProxy(item,
             ingress: getIngress(env, item),
             egress: [
                 ...(needs.length > 0
-                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: { [ID_LABEL]: need.service.id() } } })) }]
+                    ? [{ to: needs.map((need) => ({ podSelector: { matchLabels: (0, resources_1.getItemLabels)(need.service, env) } })) }]
                     : []),
                 ...(proxied
                     ? [
@@ -306,12 +312,14 @@ async function releaseKubernetesItemNetwork(instance, env, item, scope) {
 }
 exports.releaseKubernetesItemNetwork = releaseKubernetesItemNetwork;
 // Every policy and proxy of the item, of any run, once none of its pods runs:
-// what `down` and `clean` remove.
+// what `down` and `clean` remove. On a server, those of the server's run only: the
+// runs of a caller share their namespace, and the pods of another run of the same item
+// would run unlimited without their policy.
 async function removeAllKubernetesItemNetworks(instance, env, item) {
-    if (!(await awaitPodsEnded(instance, env, { [ID_LABEL]: item.id() }))) {
+    if (!(await awaitPodsEnded(instance, env, (0, resources_1.getItemLabels)(item, env)))) {
         throw new Error(`pods of ${item.name} are still running, so its network policy stays`);
     }
-    const labelSelector = `${NETWORK_OF_LABEL}=${item.id()}`;
+    const labelSelector = toSelector({ [NETWORK_OF_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) });
     let policies;
     try {
         const list = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', env.namespace, undefined, undefined, undefined, undefined, labelSelector);

package/dist/kubernetes/remove-persistent-data.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.removePersistentData = void 0;
+const resources_1 = require("./resources");
 async function removePersistentData(instance, kubernetes, task) {
-    const labelSelector = `hammerkit.dev/id=${task.id()}`;
+    // on a server, the run's: a claim it kept has no run label and goes once it expired
+    const labelSelector = (0, resources_1.getItemSelector)(task, kubernetes);
     const configmaps = await instance.coreApi.listNamespacedConfigMap({ namespace: kubernetes.namespace, labelSelector });
     for (const configMap of configmaps.items) {
         if (configMap.metadata?.name) {

package/dist/kubernetes/remove-stale-items.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getStaleItemSelector = exports.removeStaleItems = void 0;
+const job_placement_1 = require("./job-placement");
+const resources_1 = require("./resources");
+const apply_1 = require("./apply");
+/**
+ * What a service run made for items it has no more, once it moved to another commit: the
+ * Deployment, Service, Secret, policies and Jobs of a service or task whose id changed (a new
+ * image, other volumes). Their claims stay, with the namespace of the run.
+ */
+async function removeStaleItems(instance, env, ids) {
+    if (!env.server) {
+        return;
+    }
+    const namespace = env.namespace;
+    const labelSelector = getStaleItemSelector(env.server.run, ids);
+    const selector = { namespace, labelSelector };
+    await instance.appsApi.deleteCollectionNamespacedDeployment({ ...selector, propagationPolicy: 'Background' });
+    await instance.batchApi.deleteCollectionNamespacedJob({ ...selector, propagationPolicy: 'Background' });
+    await instance.coreApi.deleteCollectionNamespacedService(selector);
+    await instance.coreApi.deleteCollectionNamespacedSecret(selector);
+    const policies = await instance.objectApi.list('networking.k8s.io/v1', 'NetworkPolicy', namespace, undefined, undefined, undefined, undefined, labelSelector);
+    for (const policy of policies.items) {
+        const metadata = { name: policy.metadata?.name, namespace };
+        await instance.objectApi
+            .delete({ apiVersion: 'networking.k8s.io/v1', kind: 'NetworkPolicy', metadata })
+            .catch((e) => {
+            if ((0, apply_1.statusCodeOf)(e) !== 404) {
+                throw e;
+            }
+        });
+    }
+}
+exports.removeStaleItems = removeStaleItems;
+// the objects of the run's items but those of `ids`
+function getStaleItemSelector(run, ids) {
+    const current = [...new Set(ids)].sort();
+    return [
+        `${job_placement_1.RUN_LABEL}=${(0, job_placement_1.getRunLabelValue)(run)}`,
+        resources_1.ID_LABEL,
+        ...(current.length > 0 ? [`${resources_1.ID_LABEL} notin (${current.join(',')})`] : []),
+    ].join(',');
+}
+exports.getStaleItemSelector = getStaleItemSelector;
+//# sourceMappingURL=remove-stale-items.js.map

package/dist/kubernetes/resources.js

@@ -1,12 +1,34 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeInvalidCharacters = exports.getResourceName = void 0;
+exports.removeInvalidCharacters = exports.getResourceName = exports.getItemSelector = exports.getItemLabels = exports.getItemKey = exports.ID_LABEL = void 0;
+const job_placement_1 = require("./job-placement");
 // An object name is a DNS label: lowercase letters, digits and `-`, at most 63
 // characters, starting and ending with a letter or digit. The id makes a name
 // unique; the item's name is there for people and is cut to fit.
 const MAX_NAME = 63;
-function getResourceName(item, suffix) {
-    const tail = `-${item.id()}${suffix ?? ''}`;
+exports.ID_LABEL = 'hammerkit.dev/id';
+// What tells the objects of an item apart in its namespace: its id, and on a server the
+// run's label too. The runs of a caller execute side by side in their namespace, and two
+// of them often run the same task (the same command on two sandboxes of a repository),
+// so each has objects of its own, which nothing of the other run finds or removes. The
+// first 8 digits of the run label tell runs apart, as they do for a run's proxies.
+function getItemKey(item, env) {
+    return env.server ? `${item.id().substring(0, 32)}-${(0, job_placement_1.getRunLabelValue)(env.server.run).substring(0, 8)}` : item.id();
+}
+exports.getItemKey = getItemKey;
+// The labels the objects of an item carry, by which they are found again
+function getItemLabels(item, env) {
+    return { [exports.ID_LABEL]: item.id(), ...(0, job_placement_1.getRunLabels)(env) };
+}
+exports.getItemLabels = getItemLabels;
+function getItemSelector(item, env) {
+    return Object.entries(getItemLabels(item, env))
+        .map(([key, value]) => `${key}=${value}`)
+        .join(',');
+}
+exports.getItemSelector = getItemSelector;
+function getResourceName(item, env, suffix) {
+    const tail = `-${getItemKey(item, env)}${suffix ?? ''}`;
     const name = removeInvalidCharacters(item.name)
         .slice(0, Math.max(0, MAX_NAME - tail.length))
         .replace(/-+$/, '');

package/dist/kubernetes/restore-kubernetes-data.js

@@ -8,14 +8,18 @@ const ensure_persistent_data_1 = require("./ensure-persistent-data");
 const claim_layout_1 = require("./claim-layout");
 // Where the upload pod (which mounts the work's claim at /dev/hammerkit/<id>) unpacks the
 // archive of an output: the output's place in the claim, which its mount reads
-function getRestoreTarget(work, path) {
-    return `/dev/hammerkit/${work.id()}/${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
+function getRestoreTarget(work, path, 
+// the folder of the state on a service run, see getStateFilesDir
+stateDir) {
+    return `/dev/hammerkit/${work.id()}/${stateDir ? `${stateDir}/` : ''}${(0, claim_layout_1.getClaimSubPath)(work, path)}`;
 }
 exports.getRestoreTarget = getRestoreTarget;
 async function restoreKubernetesData(work, kubernetes, instance, environment, path, 
-// the kept claim of the work holds what an earlier run left, which goes first
+// the kept claim of the work holds what an earlier run left, which goes first; on a
+// service run, the outputs go into the folder of `stateKey` (see getStateFilesDir)
 options = {}) {
-    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes);
+    const persistence = await (0, volumes_1.getKubernetesPersistence)(work, kubernetes, options.stateKey);
+    const stateDir = kubernetes.server?.serviceRun ? (0, volumes_1.getItemFilesDir)(work, options.stateKey) : undefined;
     await (0, ensure_persistent_data_1.getPodForPersistence)(instance, kubernetes, work, persistence, 'write', async (name) => {
         if (options.reset) {
             await (0, ensure_persistent_data_1.clearClaim)(instance, kubernetes, work, name);
@@ -24,7 +28,7 @@ options = {}) {
             if (!(await environment.file.exists(generatedArchive.filename))) {
                 continue;
             }
-            const targetPath = getRestoreTarget(work, generatedArchive.path);
+            const targetPath = getRestoreTarget(work, generatedArchive.path, stateDir);
             // a claim made for this run (a server removes the claims of a run when it
             // ends) does not have the directory of the output yet
             let failure = null;