@grnsft/if 0.5.0 → 0.6.0

package/manifests/examples/pipelines/pipeline-teads-sci.yml

@@ -79,17 +79,17 @@ tree:
   children:
     child-1:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - sci
-      config:
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800

package/manifests/examples/pipelines/pipeline-with-aggregate.yml

@@ -3,25 +3,51 @@ description: a full pipeline with the aggregate feature enabled
 tags:
 aggregation:
   metrics:
-    - "carbon"
+    - carbon
   type: "both"
 initialize:
   plugins:
     "interpolate":
       method: Interpolation
-      path: 'builtin'
+      path: "builtin"
       global-config:
         method: linear
         x: [0, 10, 50, 100]
         y: [0.12, 0.32, 0.75, 1.02]
-        input-parameter: 'cpu/utilization'
-        output-parameter: 'cpu-factor'
+        input-parameter: "cpu/utilization"
+        output-parameter: "cpu-factor"
+      parameter-metadata:
+        inputs:
+          cpu/utilization:
+            unit: percentage
+            description: refers to CPU utilization.
+            aggregation-method: avg
+        outputs:
+          cpu-factor:
+            unit: kWh
+            description: result of interpolate
+            aggregation-method: avg
     "cpu-factor-to-wattage":
       method: Multiply
       path: builtin
       global-config:
         input-parameters: ["cpu-factor", "cpu/thermal-design-power"]
         output-parameter: "cpu-wattage"
+      parameter-metadata:
+        inputs:
+          cpu-factor:
+            unit: kWh
+            description: result of interpolate
+            aggregation-method: avg
+          cpu/thermal-design-power:
+            unit: kWh
+            description: thermal design power for a processor
+            aggregation-method: avg
+        outputs:
+          cpu-wattage:
+            unit: kWh
+            description: the energy used by the CPU
+            aggregation-method: sum
     "wattage-times-duration":
       method: Multiply
       path: builtin
@@ -35,6 +61,17 @@ initialize:
         numerator: cpu-wattage-times-duration
         denominator: 3600000
         output: cpu-energy-raw
+      parameter-metadata:
+        inputs:
+          cpu-wattage-times-duration:
+            unit: kWh
+            description: CPU wattage multiplied by duration
+            aggregation-method: sum
+        outputs:
+          cpu-energy-raw:
+            unit: kWh
+            description: Raw energy used by CPU in kWh
+            aggregation-method: sum
     "calculate-vcpu-ratio":
       method: Divide
       path: "builtin"
@@ -42,6 +79,21 @@ initialize:
         numerator: vcpus-total
         denominator: vcpus-allocated
         output: vcpu-ratio
+      parameter-metadata:
+        inputs:
+          vcpus-total:
+            unit: count
+            description: total number of vcpus available on a particular resource
+            aggregation-method: none
+          vcpus-allocated:
+            unit: count
+            description: number of vcpus allocated to particular resource
+            aggregation-method: none
+        outputs:
+          vcpu-ratio:
+            unit: none
+            description: Ratio of vCPUs
+            aggregation-method: none
     "correct-cpu-energy-for-vcpu-ratio":
       method: Divide
       path: "builtin"
@@ -58,11 +110,32 @@ initialize:
       global-config:
         input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"]
         output-parameter: "carbon-operational"
+      parameter-metadata:
+        inputs:
+          cpu-energy-kwh:
+            unit: kWh
+            description: Corrected CPU energy in kWh
+            aggregation-method: sum
+          grid/carbon-intensity:
+            unit: gCO2eq/kWh
+            description: Carbon intensity for the grid
+            aggregation-method: avg
+        outputs:
+          carbon-operational:
+            unit: gCO2eq
+            description: Operational carbon footprint
+            aggregation-method: sum
     "sci":
       path: "builtin"
       method: Sci
       global-config:
         functional-unit: requests # factor to convert per time to per f.unit
+      parameter-metadata:
+        inputs:
+          requests:
+            unit: none
+            description: expressed the final SCI value
+            aggregation-method: sum
     "sum-carbon":
       path: "builtin"
       method: Sum
@@ -71,6 +144,12 @@ initialize:
           - carbon-operational
           - carbon-embodied
         output-parameter: carbon
+      parameter-metadata:
+        outputs:
+          carbon:
+            unit: gCO2eq
+            description: product of carbon
+            aggregation-method: sum
     "time-sync":
       method: TimeSync
       path: "builtin"
@@ -83,22 +162,21 @@ tree:
   children:
     child-1:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - cloud/instance-type
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -134,22 +212,21 @@ tree:
           requests: 30
     child-2:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - cloud/instance-type
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -182,4 +259,4 @@ tree:
           cpu/utilization: 33
           cloud/instance-type: A1
           cloud/region: uk-west
-          requests: 180
+          requests: 180

package/manifests/examples/pipelines/pipeline-with-mocks.yml

@@ -3,7 +3,7 @@ description: a full pipeline seeded with data from mock-observations feature
 tags:
 aggregation:
   metrics:
-    - "carbon"
+    - carbon
   type: "both"
 initialize:
   plugins:
@@ -12,71 +12,185 @@ initialize:
       method: MockObservations
       path: "builtin"
       global-config:
-        timestamp-from: 2023-07-06T00:00
-        timestamp-to: 2023-07-06T00:10
-        duration: 60
+        timestamp-from: "2023-12-12T00:00:00.000Z"
+        timestamp-to: "2023-12-12T00:00:13.000Z"
+        duration: 30
         components:
           - cloud/instance-type: A1
         generators:
           common:
             cloud/region: uk-west
-            common-key: common-val
           randint:
             cpu/utilization:
               min: 1
               max: 99
-    "interpolate":
+      parameter-metadata:
+        inputs:
+          timestamp:
+            description: refers to the time of occurrence of the input
+            unit: RFC3339
+            aggregation-method: none
+          duration:
+            description: refers to the duration of the input
+            unit: seconds
+            aggregation-method: sum
+          cloud/instance-type:
+            description: type of Cloud Instance name used in the cloud provider APIs
+            unit: none
+            aggregation-method: none
+          cloud/region:
+            description: region cloud instance
+            unit: none
+            aggregation-method: none
+    interpolate:
       method: Interpolation
-      path: 'builtin'
+      path: "builtin"
       global-config:
         method: linear
         x: [0, 10, 50, 100]
         y: [0.12, 0.32, 0.75, 1.02]
-        input-parameter: 'cpu/utilization'
-        output-parameter: 'cpu-factor'
-    "cpu-factor-to-wattage":
+        input-parameter: "cpu/utilization"
+        output-parameter: "cpu-factor"
+      parameter-metadata:
+        inputs:
+          cpu/utilization:
+            description: refers to CPU utilization.
+            unit: percentage
+            aggregation-method: avg
+        outputs:
+          cpu-factor:
+            description: result of interpolate
+            unit: kWh
+            aggregation-method: avg
+    cpu-factor-to-wattage:
       method: Multiply
       path: builtin
       global-config:
         input-parameters: ["cpu-factor", "cpu/thermal-design-power"]
         output-parameter: "cpu-wattage"
-    "wattage-times-duration":
+      parameter-metadata:
+        inputs:
+          cpu-factor:
+            description: result of interpolate
+            unit: kWh
+            aggregation-method: avg
+          cpu/thermal-design-power:
+            description: thermal design power for a processor
+            unit: kWh
+            aggregation-method: avg
+        outputs:
+          cpu-wattage:
+            description: the energy used by the CPU
+            unit: kWh
+            aggregation-method: sum
+    wattage-times-duration:
       method: Multiply
       path: builtin
       global-config:
         input-parameters: ["cpu-wattage", "duration"]
         output-parameter: "cpu-wattage-times-duration"
-    "wattage-to-energy-kwh":
+      parameter-metadata:
+        inputs:
+          cpu-wattage:
+            description: Energy used by the CPU
+            unit: kWh
+            aggregation-method: sum
+          duration:
+            description: Duration of the observation
+            unit: seconds
+            aggregation-method: sum
+        outputs:
+          cpu-wattage-times-duration:
+            description: CPU wattage multiplied by duration
+            unit: kWh
+            aggregation-method: sum
+    wattage-to-energy-kwh:
       method: Divide
       path: "builtin"
       global-config:
         numerator: cpu-wattage-times-duration
         denominator: 3600000
         output: cpu-energy-raw
-    "calculate-vcpu-ratio":
+      parameter-metadata:
+        inputs:
+          cpu-wattage-times-duration:
+            description: CPU wattage multiplied by duration
+            unit: kWh
+            aggregation-method: sum
+        outputs:
+          cpu-energy-raw:
+            description: Raw energy used by CPU in kWh
+            unit: kWh
+            aggregation-method: sum
+    calculate-vcpu-ratio:
       method: Divide
       path: "builtin"
       global-config:
         numerator: vcpus-total
         denominator: vcpus-allocated
         output: vcpu-ratio
-    "correct-cpu-energy-for-vcpu-ratio":
+      parameter-metadata:
+        inputs:
+          vcpus-total:
+            description: total number of vcpus available on a particular resource
+            unit: count
+            aggregation-method: none
+          vcpus-allocated:
+            description: number of vcpus allocated to particular resource
+            unit: count
+            aggregation-method: none
+        outputs:
+          vcpu-ratio:
+            description: Ratio of vCPUs
+            unit: none
+            aggregation-method: none
+    correct-cpu-energy-for-vcpu-ratio:
       method: Divide
       path: "builtin"
       global-config:
         numerator: cpu-energy-raw
         denominator: vcpu-ratio
         output: cpu-energy-kwh
-    "sci-embodied":
+      parameter-metadata:
+        inputs:
+          cpu-energy-raw:
+            description: Raw energy used by CPU in kWh
+            unit: kWh
+            aggregation-method: sum
+          vcpu-ratio:
+            description: Ratio of vCPUs
+            unit: none
+            aggregation-method: none
+        outputs:
+          cpu-energy-kwh:
+            description: Corrected CPU energy in kWh
+            unit: kWh
+            aggregation-method: sum
+    sci-embodied:
       path: "builtin"
       method: SciEmbodied
-    "operational-carbon":
+    operational-carbon:
       method: Multiply
       path: builtin
       global-config:
         input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"]
         output-parameter: "carbon-operational"
-    "sum-carbon":
+      parameter-metadata:
+        inputs:
+          cpu-energy-kwh:
+            description: Corrected CPU energy in kWh
+            unit: kWh
+            aggregation-method: sum
+          grid/carbon-intensity:
+            description: Carbon intensity for the grid
+            unit: gCO2eq/kWh
+            aggregation-method: avg
+        outputs:
+          carbon-operational:
+            description: Operational carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+    sum-carbon:
       path: "builtin"
       method: Sum
       global-config:
@@ -84,12 +198,38 @@ initialize:
           - carbon-operational
           - carbon-embodied
         output-parameter: carbon
-    "sci":
+      parameter-metadata:
+        inputs:
+          carbon-operational:
+            description: Operational carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+          carbon-embodied:
+            description: Embodied carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+        outputs:
+          carbon:
+            description: Total carbon footprint
+            unit: gCO2eq
+            aggregation-method: sum
+    sci:
       path: "builtin"
       method: Sci
       global-config:
         functional-unit: "requests"
-    "time-sync":
+      parameter-metadata:
+        inputs:
+          requests:
+            description: expressed the final SCI value
+            unit: none
+            aggregation-method: sum
+        outputs:
+          sci:
+            description: Scientific Carbon Intensity
+            unit: none
+            aggregation-method: none
+    time-sync:
       method: TimeSync
       path: "builtin"
       global-config:
@@ -97,30 +237,27 @@ initialize:
         end-time: "2023-12-12T00:01:00.000Z"
         interval: 5
         allow-padding: true
-    "group-by":
-      path: builtin
-      method: GroupBy
-  # outputs: ['yaml']
 tree:
   children:
     child-1:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - instance-type
+        observe:
+          - mock-observations
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -130,48 +267,25 @@ tree:
         vcpus-total: 8
         vcpus-allocated: 1
       inputs:
-        - timestamp: "2023-12-12T00:00:00.000Z"
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          duration: 1
-          cpu/utilization: 10
-          requests: 30
-        - timestamp: "2023-12-12T00:00:01.000Z"
-          duration: 5
-          cpu/utilization: 20
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 40
-        - timestamp: "2023-12-12T00:00:06.000Z"
-          duration: 7
-          cpu/utilization: 15
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 30
-        - timestamp: "2023-12-12T00:00:13.000Z"
-          duration: 30
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          cpu/utilization: 15
-          requests: 50
     child-2:
       pipeline:
-        - interpolate
-        - cpu-factor-to-wattage
-        - wattage-times-duration
-        - wattage-to-energy-kwh
-        - calculate-vcpu-ratio
-        - correct-cpu-energy-for-vcpu-ratio
-        - sci-embodied
-        - operational-carbon
-        - sum-carbon
-        - time-sync
-        - sci
-      config:
-        group-by:
-          group:
-            - cloud/region
-            - cloud/instance-type
+        observe:
+          - mock-observations
+        regroup:
+          - cloud/region
+          - cloud/instance-type
+        compute:
+          - interpolate
+          - cpu-factor-to-wattage
+          - wattage-times-duration
+          - wattage-to-energy-kwh
+          - calculate-vcpu-ratio
+          - correct-cpu-energy-for-vcpu-ratio
+          - sci-embodied
+          - operational-carbon
+          - sum-carbon
+          - time-sync
+          - sci
       defaults:
         cpu/thermal-design-power: 100
         grid/carbon-intensity: 800
@@ -181,27 +295,3 @@ tree:
         vcpus-total: 8
         vcpus-allocated: 1
       inputs:
-        - timestamp: "2023-12-12T00:00:00.000Z"
-          duration: 1
-          cpu/utilization: 30
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 30
-        - timestamp: "2023-12-12T00:00:01.000Z"
-          duration: 5
-          cpu/utilization: 28
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 40
-        - timestamp: "2023-12-12T00:00:06.000Z"
-          duration: 7
-          cpu/utilization: 40
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 50
-        - timestamp: "2023-12-12T00:00:13.000Z"
-          duration: 30
-          cpu/utilization: 33
-          cloud/instance-type: A1
-          cloud/region: uk-west
-          requests: 60

package/manifests/examples/pipelines/scenario-1.yml

@@ -0,0 +1,34 @@
+name: demo
+description: demo for observe feat
+tags:
+initialize:
+  plugins:
+    mock-observations:
+      kind: plugin
+      method: MockObservations
+      path: "builtin"
+      global-config:
+        timestamp-from: 2023-07-06T00:00
+        timestamp-to: 2023-07-06T00:01
+        duration: 60
+        components:
+          - cloud/instance-type: A1
+          - cloud/instance-type: B1
+        generators:
+          common:
+            region: uk-west
+            common-key: common-val
+          randint:
+            cpu/utilization:
+              min: 1
+              max: 99
+            memory/utilization:
+              min: 1
+              max: 99
+tree:
+  children:
+    child:
+      pipeline:
+        observe:
+          - mock-observations
+      inputs: null